<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/7/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/7/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9634.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9634.html" class="post-title-link" itemprop="url">Scrapy 如何存储图片？</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p><a href="https://docs.scrapy.org/en/latest/topics/item-pipeline.html#take-screenshot-of-item" target="_blank" rel="noopener">官方文档说明</a></p>
                  <h3 id="在设置中找到ITEM-PIPELINES并加入以下代码"><a href="#在设置中找到ITEM-PIPELINES并加入以下代码" class="headerlink" title="在设置中找到ITEM_PIPELINES并加入以下代码"></a>在设置中找到ITEM_PIPELINES并加入以下代码</h3>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">scrapy</span><span class="selector-class">.pipelines</span><span class="selector-class">.images</span><span class="selector-class">.ImagesPipeline</span>: 301</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="settings配置"><a href="#settings配置" class="headerlink" title="settings配置:"></a>settings配置:</h3>
                  <h4 id="图片存储路径："><a href="#图片存储路径：" class="headerlink" title="图片存储路径："></a>图片存储路径：</h4>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">IMAGES_STORE</span> = “your path”</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="图片存储天数"><a href="#图片存储天数" class="headerlink" title="图片存储天数"></a>图片存储天数</h4>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">images_EXPIRES</span> =  <span class="number">30</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="设置缩略图-固定值"><a href="#设置缩略图-固定值" class="headerlink" title="设置缩略图(固定值):"></a>设置缩略图(固定值):</h4>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">IMAGES_THUMBS = &#123;</span><br><span class="line"><span class="string">'small'</span>:(<span class="number">50</span>,<span class="number">50</span>)</span><br><span class="line"><span class="string">'big'</span>:(<span class="number">270</span>,<span class="number">270</span>)</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># 配置图片管道参数</span><br><span class="line">BASE_DIR = <span class="built_in">os</span>.<span class="built_in">path</span>.dirname(<span class="built_in">os</span>.<span class="built_in">path</span>.dirname(<span class="built_in">os</span>.<span class="built_in">path</span>.abspath(__file__)))</span><br><span class="line">IMAGES_STORE = <span class="built_in">os</span>.<span class="built_in">path</span>.join(BASE_DIR,<span class="string">'images'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <blockquote>
                    <figure class="highlight lua">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"># 寻找此文件的父级目录</span><br><span class="line"><span class="built_in">os</span>.<span class="built_in">path</span>.dirname() </span><br><span class="line"># 当前脚本的绝对路径目录</span><br><span class="line"><span class="built_in">os</span>.<span class="built_in">path</span>.abspath(__file__)</span><br><span class="line"># __file__当前脚本的名字 </span><br><span class="line"></span><br><span class="line">IMAGES_STORE = <span class="built_in">os</span>.<span class="built_in">path</span>.join(BASE_DIR,<span class="string">'images'</span>)</span><br><span class="line">将BASE_DIR新增IMAGES文件夹路径</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                  </blockquote>
                  <p>设置spider中获取images_url的提取方法</p>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">item[<span class="string">'image_urls'</span>] = <span class="string">"提取语法"</span></span><br><span class="line"># item[<span class="string">'image_urls'</span>] = response.css(<span class="string">".pic img:attr('src')"</span>).extract()</span><br><span class="line">item[<span class="string">'images'</span>] = [] # 【】中不需要填写,下载图片之后，保存本地的文件位置</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <blockquote>
                    <p>使用ImagesPipeline下载图片时，需要使用images_urls字段，images_urls一般是可迭代的列表或元组类型</p>
                  </blockquote>
                  <p>如果遇到图片反扒请打开</p>
                  <figure class="highlight autoit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta"># DEFAULT_REQUEST_HEADERS = &#123;</span></span><br><span class="line"><span class="meta">#   <span class="string">'Accept'</span>: <span class="string">'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'</span>,</span></span><br><span class="line"><span class="meta">#   <span class="string">'Accept-Language'</span>: <span class="string">'en'</span>,</span></span><br><span class="line"><span class="meta">#    <span class="string">"referer"</span>:<span class="string">"自行配置"</span></span></span><br><span class="line"><span class="meta"># &#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>存入MongoDB，示例代码</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"> import pymongo</span><br><span class="line">from itemadapter import ItemAdapter</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MongoPipeline</span>:</span></span><br><span class="line"></span><br><span class="line">    collection_name = <span class="string">'scrapy_items'</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, mongo_uri, mongo_db)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.mongo_uri = mongo_uri</span><br><span class="line">        <span class="keyword">self</span>.mongo_db = mongo_db</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> cls(</span><br><span class="line">            mongo_uri=crawler.settings.get(<span class="string">'MONGO_URI'</span>),</span><br><span class="line">            mongo_db=crawler.settings.get(<span class="string">'MONGO_DATABASE'</span>, <span class="string">'items'</span>)</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client = pymongo.MongoClient(<span class="keyword">self</span>.mongo_uri)</span><br><span class="line">        <span class="keyword">self</span>.db = <span class="keyword">self</span>.client[<span class="keyword">self</span>.mongo_db]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.db[<span class="keyword">self</span>.collection_name].insert_one(ItemAdapter(item).asdict())</span><br><span class="line">        <span class="keyword">return</span> item</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>更多详情请查阅官方文档：<a href="https://docs.scrapy.org/en/latest/topics/item-pipeline.html#take-screenshot-of-item" target="_blank" rel="noopener">https://docs.scrapy.org/en/latest/topics/item-pipeline.html#take-screenshot-of-item</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-28 22:20:26" itemprop="dateCreated datePublished" datetime="2020-08-28T22:20:26+08:00">2020-08-28</time>
                </span>
                <span id="/9634.html" class="post-meta-item leancloud_visitors" data-flag-title="Scrapy 如何存储图片？" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.6k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9591.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> JavaScript <i class="label-arrow"></i>
                  </a>
                  <a href="/9591.html" class="post-title-link" itemprop="url">某道词典在线翻译JS解密,完全扣js代码解密 （小白基础篇）</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>一、先试着英汉翻译一波：</p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812195418734.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p><strong>1.按F12打开调试台，再点击Network,再点击Headers,可以找到i=good,这就是我们刚才输入需要翻译的词语good,from Data中的就是请求的参数，分别为：</strong></p>
                  <p><strong>i: good from: AUTO to: AUTO smartresult: dict client: fanyideskweb salt: 15972332870677 sign: 3a078c10344e67f95822ae9389e1363f lts: 1597233287067 bv: 85c050fb1c0b4d824d801d079db7371a doctype: json version: 2.1 keyfrom: fanyi.web action: FY_BY_CLICKBUTTION</strong></p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812204532842.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p>2.在来翻译一个新的词语，看下这些参数有无变化</p>
                  <p><img src="https://img-blog.csdnimg.cn/2020081220512561.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""><strong>i: 我是中国人 from: AUTO to: AUTO smartresult: dict client: fanyideskweb salt: 15972365771410 sign: 744ebb7fd625f1d4d9d6a270e98536c2 lts: 1597236577141 bv: 85c050fb1c0b4d824d801d079db7371a doctype: json version: 2.1 keyfrom: fanyi.web action: FY_BY_CLICKBUTTION</strong></p>
                  <p><strong>发现有5个参数是变化的（实际是4 个），分别为：</strong></p>
                  <p><strong>i: salt: sign: lts: bv:</strong> </p>
                  <p>3.看看URL，#<a href="http://fanyi.youdao.com/translate_o?smartresult=dict&amp;smartresult=rule" target="_blank" rel="noopener">http://fanyi.youdao.com/translate_o?smartresult=dict&amp;smartresult=rule</a>（） 请求连接：<a href="http://fanyi.youdao.com/translate?smartresult=dict&amp;smartresult=rule" target="_blank" rel="noopener">http://fanyi.youdao.com/translate?smartresult=dict&amp;smartresult=rule</a> 去掉<a href="http://fanyi.youdao.com/translate_o?smartresult=dict&amp;smartresult=rule" target="_blank" rel="noopener">translate_o</a>中的<a href="http://fanyi.youdao.com/translate_o?smartresult=dict&amp;smartresult=rule" target="_blank" rel="noopener">_o</a>，不去的话，轻轻就数据结果：某道词典 ，英语翻译汉语正常，汉语翻译英语时，出现{‘errorCode’: 50} ，post请求方式</p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812210040845.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p>4.点击Initiator,可以看到所有的js文件都是@fanyi.min.js:1</p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812210226824.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p>5、点击@fanyi.min.js:1进入，</p>
                  <p>点击中间的{ },格式化一下。</p>
                  <p>再找到XHR/fetch Breakpoints，添加断点， 你可以针对某一个请求或者请求的关键字设置断点：</p>
                  <p>再点一下‘+’输入url即可url为<a href="http://fanyi.youdao.com/translate_o?smartresult=dict&amp;smartresult=rule" target="_blank" rel="noopener">http://fanyi.youdao.com/translate_o?smartresult=dict&amp;smartresult=rule</a></p>
                  <p>最后再去点击一下翻译按钮</p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812210938804.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p>6.翻译：我是中国人，点击t-translate,再点击Scope，再点击Local，再点击r，即可看到我们需要的参数，在吧鼠标放到中间的r上，r = v.generateSaltSign(n)，就会弹出Object框。就是我们需要的参数。</p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812212126284.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p>7.可以看到8941行，鼠标放到v.gen erateSaltSign(n)上，会弹出f r(e)函数，点击这个函数会进入第8368行，可以看到我们需要的参数</p>
                  <p><strong>salt: sign: lts: bv:</strong> </p>
                  <p><strong>在8378行打上断点，再次点击翻译按钮，即可看多所有的参数值显示出来。</strong></p>
                  <p><img src="https://img-blog.csdnimg.cn/2020081221295839.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""><img src="image/gif;base64,R0lGODlhAQABAPABAP///wAAACH5BAEKAAAALAAAAAABAAEAAAICRAEAOw==" alt="" title="点击并拖拽以移动">​</p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812214059789.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p>8.对于第一个 ts: r,相当于pythonzhong 的ts=r赋值， 可以看到r = “” + (new Date).getTime()，知道了r,就知道了ts,r是js中 new Date().getTime()得到的是毫秒数，也就是时间戳，单位为毫秒,13位数字的字符串。</p>
                  <p>即可用python中的时间戳构造：</p>
                  <h1 id="13位时间戳获取方法：单位：毫秒-t1-time-time-ts-int-t1-1000-ts-r"><a href="#13位时间戳获取方法：单位：毫秒-t1-time-time-ts-int-t1-1000-ts-r" class="headerlink" title="13位时间戳获取方法：单位：毫秒 t1 = time.time() ts= int(t1 * 1000) ts=r"></a>13位时间戳获取方法：单位：毫秒 t1 = time.time() ts= int(t1 * 1000) ts=r</h1>
                  <p>第二个参数：salt: i，相当于python中的salt= i赋值，i = r + parseInt(10 * Math.random(), 10)，意思是随机产生一个整数 范围是0-9里面的一个随机数</p>
                  <p>ts=r,转化为字符转，parseInt(10 * Math.random(), 10)产生的随机数也转化为字符串，最后进行字符串拼接，而不是数字相加。请注意一下。跟实际的赋值对比，产生了一个14位数字的字符串，跟实际情况一样。</p>
                  <p>即可用python中的构造法： salt= str(int(ts))+str(random.randint(0,10)) print(salt)</p>
                  <p>第三个参数：bv: t,相当于python中的bv=t赋值, var t = n.md5(navigator.appVersion),navigator.appVersion的值竟然是一个 User-Agent，那么重点来了，网上教程都是md5加密相关都是直接用python原生来生成，而我就选择直接扣代码。用<a href="http://www.baidu.com/link?url=12C70-juYx55gS5XL_V1pkIyYMUXTnCc2Qo6yi4dzFFmD0sUvca_rQ4cj6APKvdyT2i4RcF2tqavgAL2xUH8dK" target="_blank" rel="noopener">python3调用<em>js</em>的库之<em>execjs</em></a> 来扣，鼠标放到var t = n.md5(navigator.appVersion)中的n.md5（）上，将会出现f md5(e)函数，点击进入，来到8196行， md5: function(e){ }，把第8196行到8278行扣下来，再应用<em><a href="http://www.baidu.com/link?url=12C70-juYx55gS5XL_V1pkIyYMUXTnCc2Qo6yi4dzFFmD0sUvca_rQ4cj6APKvdyT2i4RcF2tqavgAL2xUH8dK" target="_blank" rel="noopener">execjs</a>解析出bv</em></p>
                  <p><em>注意：自调函数调用写法：</em> </p>
                  <h1 id="var-aa-function-e-和function-aa-e-写法都可以，一样的"><a href="#var-aa-function-e-和function-aa-e-写法都可以，一样的" class="headerlink" title="var aa=function(e){ }和function  aa(e){ }写法都可以，一样的"></a>var aa=function(e){ }和function aa(e){ }写法都可以，一样的</h1>
                  <p>import execjs f = open(r”text.js”,encoding=’utf-8’).read() ctx1 = execjs.compile(f) bv=ctx1.call(‘md5_1’,ua.random) print(bv)</p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812224021674.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812221040482.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p>扣出来的js代码：主体为function md5<em>1(e) { } ，应用</em><a href="http://www.baidu.com/link?url=12C70-juYx55gS5XL_V1pkIyYMUXTnCc2Qo6yi4dzFFmD0sUvca_rQ4cj6APKvdyT2i4RcF2tqavgAL2xUH8dK" target="_blank" rel="noopener">execjs</a>解析，缺什么参数找什么参数，即可_</p>
                  <p><code>var n = function(e, t) { return e &lt;&lt; t | e &gt;&gt;&gt; 32 - t } , r = function(e, t) { var n, r, i, a, o; return i = 2147483648 &amp; e, a = 2147483648 &amp; t, n = 1073741824 &amp; e, r = 1073741824 &amp; t, o = (1073741823 &amp; e) + (1073741823 &amp; t), n &amp; r ? 2147483648 ^ o ^ i ^ a : n | r ? 1073741824 &amp; o ? 3221225472 ^ o ^ i ^ a : 1073741824 ^ o ^ i ^ a : o ^ i ^ a } , i = function(e, t, n) { return e &amp; t | ~e &amp; n } , a = function(e, t, n) { return e &amp; n | t &amp; ~n } , o = function(e, t, n) { return e ^ t ^ n } , s = function(e, t, n) { return t ^ (e | ~n) } , l = function(e, t, a, o, s, l, c) { return e = r(e, r(r(i(t, a, o), s), c)), r(n(e, l), t) } , c = function(e, t, i, o, s, l, c) { return e = r(e, r(r(a(t, i, o), s), c)), r(n(e, l), t) } , u = function(e, t, i, a, s, l, c) { return e = r(e, r(r(o(t, i, a), s), c)), r(n(e, l), t) } , d = function(e, t, i, a, o, l, c) { return e = r(e, r(r(s(t, i, a), o), c)), r(n(e, l), t) } , f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 * ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 * 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 * 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a } , p = function(e) { var t, n = &quot;&quot;, r = &quot;&quot;; for (t = 0; t &lt;= 3; t++) n += (r = &quot;0&quot; + (e &gt;&gt;&gt; 8 * t &amp; 255).toString(16)).substr(r.length - 2, 2); return n }, h = function(e) { e = e.replace(/\\x0d\\x0a/g, &quot;\\n&quot;); for (var t = &quot;&quot;, n = 0; n &lt; e.length; n++) { var r = e.charCodeAt(n); if (r &lt; 128) t += String.fromCharCode(r); else if (r &gt; 127 &amp;&amp; r &lt; 2048) t += String.fromCharCode(r &gt;&gt; 6 | 192), t += String.fromCharCode(63 &amp; r | 128); else if (r &gt;= 55296 &amp;&amp; r &lt;= 56319) { if (n + 1 &lt; e.length) { var i = e.charCodeAt(n + 1); if (i &gt;= 56320 &amp;&amp; i &lt;= 57343) { var a = 1024 * (r - 55296) + (i - 56320) + 65536; t += String.fromCharCode(240 | a &gt;&gt; 18 &amp; 7), t += String.fromCharCode(128 | a &gt;&gt; 12 &amp; 63), t += String.fromCharCode(128 | a &gt;&gt; 6 &amp; 63), t += String.fromCharCode(128 | 63 &amp; a), n++ } } } else t += String.fromCharCode(r &gt;&gt; 12 | 224), t += String.fromCharCode(r &gt;&gt; 6 &amp; 63 | 128), t += String.fromCharCode(63 &amp; r | 128) } return t }; f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 * ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 * 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 * 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a } function md5_1(e) { var t, n, i, a, o, s, m, g, v, y = Array(); for (e = e, y = f, s = 1732584193, m = 4023233417, g = 2562383102, v = 271733878, t = 0; t &lt; y.length; t += 16) n = s, i = m, a = g, o = v, s = l(s, m, g, v, y[t + 0], 7, 3614090360), v = l(v, s, m, g, y[t + 1], 12, 3905402710), g = l(g, v, s, m, y[t + 2], 17, 606105819), m = l(m, g, v, s, y[t + 3], 22, 3250441966), s = l(s, m, g, v, y[t + 4], 7, 4118548399), v = l(v, s, m, g, y[t + 5], 12, 1200080426), g = l(g, v, s, m, y[t + 6], 17, 2821735955), m = l(m, g, v, s, y[t + 7], 22, 4249261313), s = l(s, m, g, v, y[t + 8], 7, 1770035416), v = l(v, s, m, g, y[t + 9], 12, 2336552879), g = l(g, v, s, m, y[t + 10], 17, 4294925233), m = l(m, g, v, s, y[t + 11], 22, 2304563134), s = l(s, m, g, v, y[t + 12], 7, 1804603682), v = l(v, s, m, g, y[t + 13], 12, 4254626195), g = l(g, v, s, m, y[t + 14], 17, 2792965006), m = l(m, g, v, s, y[t + 15], 22, 1236535329), s = c(s, m, g, v, y[t + 1], 5, 4129170786), v = c(v, s, m, g, y[t + 6], 9, 3225465664), g = c(g, v, s, m, y[t + 11], 14, 643717713), m = c(m, g, v, s, y[t + 0], 20, 3921069994), s = c(s, m, g, v, y[t + 5], 5, 3593408605), v = c(v, s, m, g, y[t + 10], 9, 38016083), g = c(g, v, s, m, y[t + 15], 14, 3634488961), m = c(m, g, v, s, y[t + 4], 20, 3889429448), s = c(s, m, g, v, y[t + 9], 5, 568446438), v = c(v, s, m, g, y[t + 14], 9, 3275163606), g = c(g, v, s, m, y[t + 3], 14, 4107603335), m = c(m, g, v, s, y[t + 8], 20, 1163531501), s = c(s, m, g, v, y[t + 13], 5, 2850285829), v = c(v, s, m, g, y[t + 2], 9, 4243563512), g = c(g, v, s, m, y[t + 7], 14, 1735328473), m = c(m, g, v, s, y[t + 12], 20, 2368359562), s = u(s, m, g, v, y[t + 5], 4, 4294588738), v = u(v, s, m, g, y[t + 8], 11, 2272392833), g = u(g, v, s, m, y[t + 11], 16, 1839030562), m = u(m, g, v, s, y[t + 14], 23, 4259657740), s = u(s, m, g, v, y[t + 1], 4, 2763975236), v = u(v, s, m, g, y[t + 4], 11, 1272893353), g = u(g, v, s, m, y[t + 7], 16, 4139469664), m = u(m, g, v, s, y[t + 10], 23, 3200236656), s = u(s, m, g, v, y[t + 13], 4, 681279174), v = u(v, s, m, g, y[t + 0], 11, 3936430074), g = u(g, v, s, m, y[t + 3], 16, 3572445317), m = u(m, g, v, s, y[t + 6], 23, 76029189), s = u(s, m, g, v, y[t + 9], 4, 3654602809), v = u(v, s, m, g, y[t + 12], 11, 3873151461), g = u(g, v, s, m, y[t + 15], 16, 530742520), m = u(m, g, v, s, y[t + 2], 23, 3299628645), s = d(s, m, g, v, y[t + 0], 6, 4096336452), v = d(v, s, m, g, y[t + 7], 10, 1126891415), g = d(g, v, s, m, y[t + 14], 15, 2878612391), m = d(m, g, v, s, y[t + 5], 21, 4237533241), s = d(s, m, g, v, y[t + 12], 6, 1700485571), v = d(v, s, m, g, y[t + 3], 10, 2399980690), g = d(g, v, s, m, y[t + 10], 15, 4293915773), m = d(m, g, v, s, y[t + 1], 21, 2240044497), s = d(s, m, g, v, y[t + 8], 6, 1873313359), v = d(v, s, m, g, y[t + 15], 10, 4264355552), g = d(g, v, s, m, y[t + 6], 15, 2734768916), m = d(m, g, v, s, y[t + 13], 21, 1309151649), s = d(s, m, g, v, y[t + 4], 6, 4149444226), v = d(v, s, m, g, y[t + 11], 10, 3174756917), g = d(g, v, s, m, y[t + 2], 15, 718787259), m = d(m, g, v, s, y[t + 9], 21, 3951481745), s = r(s, n), m = r(m, i), g = r(g, a), v = r(v, o); return (p(s) + p(m) + p(g) + p(v)).toLowerCase() }</code></p>
                  <p>第四个参数：sign: n.md5(“fanyideskweb” + e + i + “]BjuETDhU)zqSxf-=B#7m”),相当于python中的sign=n.md5(“fanyideskweb” + e + i + “]BjuETDhU)zqSxf-=B#7m”), salt=i,e是你输入需要翻译的词语，即可写成sign=n.md5(“fanyideskweb” +str(e) + str(salt) + “]BjuETDhU)zqSxf-=B#7m”),继续选择直接扣代码。用<a href="http://www.baidu.com/link?url=12C70-juYx55gS5XL_V1pkIyYMUXTnCc2Qo6yi4dzFFmD0sUvca_rQ4cj6APKvdyT2i4RcF2tqavgAL2xUH8dK" target="_blank" rel="noopener">python3调用<em>js</em>的库之<em>execjs</em></a> 来扣，鼠标放到 sign: n.md5(“fanyideskweb” + e + i + “]BjuETDhU)zqSxf-=B#7m”)中的n.md5（）上，将会出现f md5(e)函数，点击进入，来到8196行， md5: function(e){ }，把第8196行到8278行扣下来，竟然和刚才的扣bv的方法相同，仅仅是参数不同而已，bv的是 n.md5(navigator.appVersion)中的navigator.appVersion参数，现在的asin的是sign=n.md5(“fanyideskweb” +str(e) + str(salt) + “]BjuETDhU)zqSxf-=B#7m”)中的”fanyideskweb” +str(e) + str(salt) + “]BjuETDhU)zqSxf-=B#7m”参数，再应用<em><a href="http://www.baidu.com/link?url=12C70-juYx55gS5XL_V1pkIyYMUXTnCc2Qo6yi4dzFFmD0sUvca_rQ4cj6APKvdyT2i4RcF2tqavgAL2xUH8dK" target="_blank" rel="noopener">execjs</a>解析出sign.</em></p>
                  <p><img src="https://img-blog.csdnimg.cn/20200812223547997.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1hpYW95ZWZvcmV2ZXI=,size_16,color_FFFFFF,t_70" alt=""></p>
                  <p><em>注意：自调函数调用写法：</em> </p>
                  <h1 id="var-aa-function-e-和function-aa-e-写法都可以，一样的-1"><a href="#var-aa-function-e-和function-aa-e-写法都可以，一样的-1" class="headerlink" title="var aa=function(e){ }和function  aa(e){ }写法都可以，一样的"></a>var aa=function(e){ }和function aa(e){ }写法都可以，一样的</h1>
                  <p>import execjs f = open(r”text.js”,encoding=’utf-8’).read() ctx1 = execjs.compile(f) bv=ctx1.call(‘md5_2’,ua.random) print(bv)</p>
                  <p>扣出来的js代码：主体为function md5<em>2(e) { } ，应用</em><a href="http://www.baidu.com/link?url=12C70-juYx55gS5XL_V1pkIyYMUXTnCc2Qo6yi4dzFFmD0sUvca_rQ4cj6APKvdyT2i4RcF2tqavgAL2xUH8dK" target="_blank" rel="noopener">execjs</a>解析，缺什么参数找什么参数，即可_</p>
                  <p><code>var n = function(e, t) { return e &lt;&lt; t | e &gt;&gt;&gt; 32 - t } , r = function(e, t) { var n, r, i, a, o; return i = 2147483648 &amp; e, a = 2147483648 &amp; t, n = 1073741824 &amp; e, r = 1073741824 &amp; t, o = (1073741823 &amp; e) + (1073741823 &amp; t), n &amp; r ? 2147483648 ^ o ^ i ^ a : n | r ? 1073741824 &amp; o ? 3221225472 ^ o ^ i ^ a : 1073741824 ^ o ^ i ^ a : o ^ i ^ a } , i = function(e, t, n) { return e &amp; t | ~e &amp; n } , a = function(e, t, n) { return e &amp; n | t &amp; ~n } , o = function(e, t, n) { return e ^ t ^ n } , s = function(e, t, n) { return t ^ (e | ~n) } , l = function(e, t, a, o, s, l, c) { return e = r(e, r(r(i(t, a, o), s), c)), r(n(e, l), t) } , c = function(e, t, i, o, s, l, c) { return e = r(e, r(r(a(t, i, o), s), c)), r(n(e, l), t) } , u = function(e, t, i, a, s, l, c) { return e = r(e, r(r(o(t, i, a), s), c)), r(n(e, l), t) } , d = function(e, t, i, a, o, l, c) { return e = r(e, r(r(s(t, i, a), o), c)), r(n(e, l), t) } , f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 * ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 * 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 * 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a } , p = function(e) { var t, n = &quot;&quot;, r = &quot;&quot;; for (t = 0; t &lt;= 3; t++) n += (r = &quot;0&quot; + (e &gt;&gt;&gt; 8 * t &amp; 255).toString(16)).substr(r.length - 2, 2); return n }, h = function(e) { e = e.replace(/\\x0d\\x0a/g, &quot;\\n&quot;); for (var t = &quot;&quot;, n = 0; n &lt; e.length; n++) { var r = e.charCodeAt(n); if (r &lt; 128) t += String.fromCharCode(r); else if (r &gt; 127 &amp;&amp; r &lt; 2048) t += String.fromCharCode(r &gt;&gt; 6 | 192), t += String.fromCharCode(63 &amp; r | 128); else if (r &gt;= 55296 &amp;&amp; r &lt;= 56319) { if (n + 1 &lt; e.length) { var i = e.charCodeAt(n + 1); if (i &gt;= 56320 &amp;&amp; i &lt;= 57343) { var a = 1024 * (r - 55296) + (i - 56320) + 65536; t += String.fromCharCode(240 | a &gt;&gt; 18 &amp; 7), t += String.fromCharCode(128 | a &gt;&gt; 12 &amp; 63), t += String.fromCharCode(128 | a &gt;&gt; 6 &amp; 63), t += String.fromCharCode(128 | 63 &amp; a), n++ } } } else t += String.fromCharCode(r &gt;&gt; 12 | 224), t += String.fromCharCode(r &gt;&gt; 6 &amp; 63 | 128), t += String.fromCharCode(63 &amp; r | 128) } return t }; f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 * ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 * 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 * 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a }</code></p>
                  <p>function md5_2(e) { var t, n, i, a, o, s, m, g, v, y = Array(); for (e = e, y = f(e), s = 1732584193, m = 4023233417, g = 2562383102, v = 271733878, t = 0; t &lt; y.length; t += 16) n = s, i = m, a = g, o = v, s = l(s, m, g, v, y[t + 0], 7, 3614090360), v = l(v, s, m, g, y[t + 1], 12, 3905402710), g = l(g, v, s, m, y[t + 2], 17, 606105819), m = l(m, g, v, s, y[t + 3], 22, 3250441966), s = l(s, m, g, v, y[t + 4], 7, 4118548399), v = l(v, s, m, g, y[t + 5], 12, 1200080426), g = l(g, v, s, m, y[t + 6], 17, 2821735955), m = l(m, g, v, s, y[t + 7], 22, 4249261313), s = l(s, m, g, v, y[t + 8], 7, 1770035416), v = l(v, s, m, g, y[t + 9], 12, 2336552879), g = l(g, v, s, m, y[t + 10], 17, 4294925233), m = l(m, g, v, s, y[t + 11], 22, 2304563134), s = l(s, m, g, v, y[t + 12], 7, 1804603682), v = l(v, s, m, g, y[t + 13], 12, 4254626195), g = l(g, v, s, m, y[t + 14], 17, 2792965006), m = l(m, g, v, s, y[t + 15], 22, 1236535329), s = c(s, m, g, v, y[t + 1], 5, 4129170786), v = c(v, s, m, g, y[t + 6], 9, 3225465664), g = c(g, v, s, m, y[t + 11], 14, 643717713), m = c(m, g, v, s, y[t + 0], 20, 3921069994), s = c(s, m, g, v, y[t + 5], 5, 3593408605), v = c(v, s, m, g, y[t + 10], 9, 38016083), g = c(g, v, s, m, y[t + 15], 14, 3634488961), m = c(m, g, v, s, y[t + 4], 20, 3889429448), s = c(s, m, g, v, y[t + 9], 5, 568446438), v = c(v, s, m, g, y[t + 14], 9, 3275163606), g = c(g, v, s, m, y[t + 3], 14, 4107603335), m = c(m, g, v, s, y[t + 8], 20, 1163531501), s = c(s, m, g, v, y[t + 13], 5, 2850285829), v = c(v, s, m, g, y[t + 2], 9, 4243563512), g = c(g, v, s, m, y[t + 7], 14, 1735328473), m = c(m, g, v, s, y[t + 12], 20, 2368359562), s = u(s, m, g, v, y[t + 5], 4, 4294588738), v = u(v, s, m, g, y[t + 8], 11, 2272392833), g = u(g, v, s, m, y[t + 11], 16, 1839030562), m = u(m, g, v, s, y[t + 14], 23, 4259657740), s = u(s, m, g, v, y[t + 1], 4, 2763975236), v = u(v, s, m, g, y[t + 4], 11, 1272893353), g = u(g, v, s, m, y[t + 7], 16, 4139469664), m = u(m, g, v, s, y[t + 10], 23, 3200236656), s = u(s, m, g, v, y[t + 13], 4, 681279174), v = u(v, s, m, g, y[t + 0], 11, 3936430074), g = u(g, v, s, m, y[t + 3], 16, 3572445317), m = u(m, g, v, s, y[t + 6], 23, 76029189), s = u(s, m, g, v, y[t + 9], 4, 3654602809), v = u(v, s, m, g, y[t + 12], 11, 3873151461), g = u(g, v, s, m, y[t + 15], 16, 530742520), m = u(m, g, v, s, y[t + 2], 23, 3299628645), s = d(s, m, g, v, y[t + 0], 6, 4096336452), v = d(v, s, m, g, y[t + 7], 10, 1126891415), g = d(g, v, s, m, y[t + 14], 15, 2878612391), m = d(m, g, v, s, y[t + 5], 21, 4237533241), s = d(s, m, g, v, y[t + 12], 6, 1700485571), v = d(v, s, m, g, y[t + 3], 10, 2399980690), g = d(g, v, s, m, y[t + 10], 15, 4293915773), m = d(m, g, v, s, y[t + 1], 21, 2240044497), s = d(s, m, g, v, y[t + 8], 6, 1873313359), v = d(v, s, m, g, y[t + 15], 10, 4264355552), g = d(g, v, s, m, y[t + 6], 15, 2734768916), m = d(m, g, v, s, y[t + 13], 21, 1309151649), s = d(s, m, g, v, y[t + 4], 6, 4149444226), v = d(v, s, m, g, y[t + 11], 10, 3174756917), g = d(g, v, s, m, y[t + 2], 15, 718787259), m = d(m, g, v, s, y[t + 9], 21, 3951481745), s = r(s, n), m = r(m, i), g = r(g, a), v = r(v, o); return (p(s) + p(m) + p(g) + p(v)).toLowerCase() }</p>
                  <p>9.大功告成，四个可变参数全部搞定，即可搞定&amp;</p>
                  <p>附上完整代码 python部分：</p>
                  <p><code>import time import execjs import random import requests import json from fake_useragent import UserAgent # var aa=function(e) function aa(e)</code></p>
                  <p>class YouDaoTanslate(): def <strong>init</strong>(self): #类方法调用 self.js_par() def js_par(self): translation_data = input(“请输入要翻译的词或者句子：”) # 13位时间戳获取方法：单位：毫秒 ua = UserAgent() print(ua.random) t1 = time.time() ts = int(t1 * 1000) print(ts) salt = str(int(ts)) + str(random.randint(0, 10)) print(salt)</p>
                  <p>f = open(r”text.js”, encoding=’utf-8’).read() ctx1 = execjs.compile(f) bv = ctx1.call(‘md5_1’, ua.random) print(bv)</p>
                  <p>ctx2 = execjs.compile(f) sign_data = ‘fanyideskweb’ + translation_data + salt + ‘mmbP%A-r6U3Nw(n]BjuEU’ sign = ctx2.call(‘md5_2’, sign_data) print(sign)</p>
                  <p>self.get_translateData(translation_data,ua,ts,salt,bv,sign) def get_translateData(self,translation_data,ua,ts,salt,bv,sign): url = ‘<a href="http://fanyi.youdao.com/translate?smartresult=dict&amp;smartresult=rule" target="_blank" rel="noopener">http://fanyi.youdao.com/translate?smartresult=dict&amp;smartresult=rule</a>‘ headers = { ‘Host’: ‘fanyi.youdao.com’, ‘Origin’: ‘<a href="http://fanyi.youdao.com" target="_blank" rel="noopener">http://fanyi.youdao.com</a>‘, ‘Referer’: ‘<a href="http://fanyi.youdao.com/" target="_blank" rel="noopener">http://fanyi.youdao.com/</a>‘, ‘User-Agent’: ua.random, ‘X-Requested-With’: ‘XMLHttpRequest’, ‘Cookie’: ‘OUTFOX_SEARCH_USER_ID_NCOO=892433278.3204294; OUTFOX_SEARCH_USER_ID=”-1911793285@10.108.160.19”; _ntes_nnid=d2bb7793f13c9a83907e33d40665337a,1597158692753; JSESSIONID=aaamUZK4O580YaKjmjIpx; <strong>_rl</strong>test__cookies=1597227802599’ } data = { ‘i’: translation_data, ‘from’: ‘AUTO’, ‘to’: ‘AUTO’, ‘smartresult’: ‘dict’, ‘client’: ‘fanyideskweb’, ‘salt’: str(salt), ‘sign’: str(sign), ‘ts’: str(ts), ‘bv’: str(bv), ‘doctype’: ‘json’, ‘version’: ‘2.1’, ‘keyfrom’: ‘fanyi.web’, ‘action’: ‘FY_BY_CLICKBUTTION’ }</p>
                  <p>response = requests.post(url=url, data=data, headers=headers)</p>
                  <p>response = requests.post(url=url, data=data, headers=headers) translate_results = json.loads(response.text) # 找到翻译结果 if ‘translateResult’ in translate_results: translate_results = translate_results[‘translateResult’][0][0][‘tgt’] print(“翻译的结果是：%s” % translate_results)</p>
                  <p>else: print(translate_results)</p>
                  <p>self.js_par()</p>
                  <p>YouDaoTanslate()</p>
                  <p>js部分:</p>
                  <p><code>var n = function(e, t) { return e &lt;&lt; t | e &gt;&gt;&gt; 32 - t } , r = function(e, t) { var n, r, i, a, o; return i = 2147483648 &amp; e, a = 2147483648 &amp; t, n = 1073741824 &amp; e, r = 1073741824 &amp; t, o = (1073741823 &amp; e) + (1073741823 &amp; t), n &amp; r ? 2147483648 ^ o ^ i ^ a : n | r ? 1073741824 &amp; o ? 3221225472 ^ o ^ i ^ a : 1073741824 ^ o ^ i ^ a : o ^ i ^ a } , i = function(e, t, n) { return e &amp; t | ~e &amp; n } , a = function(e, t, n) { return e &amp; n | t &amp; ~n } , o = function(e, t, n) { return e ^ t ^ n } , s = function(e, t, n) { return t ^ (e | ~n) } , l = function(e, t, a, o, s, l, c) { return e = r(e, r(r(i(t, a, o), s), c)), r(n(e, l), t) } , c = function(e, t, i, o, s, l, c) { return e = r(e, r(r(a(t, i, o), s), c)), r(n(e, l), t) } , u = function(e, t, i, a, s, l, c) { return e = r(e, r(r(o(t, i, a), s), c)), r(n(e, l), t) } , d = function(e, t, i, a, o, l, c) { return e = r(e, r(r(s(t, i, a), o), c)), r(n(e, l), t) } , f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 * ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 * 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 * 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a } , p = function(e) { var t, n = &quot;&quot;, r = &quot;&quot;; for (t = 0; t &lt;= 3; t++) n += (r = &quot;0&quot; + (e &gt;&gt;&gt; 8 * t &amp; 255).toString(16)).substr(r.length - 2, 2); return n }, h = function(e) { e = e.replace(/\\x0d\\x0a/g, &quot;\\n&quot;); for (var t = &quot;&quot;, n = 0; n &lt; e.length; n++) { var r = e.charCodeAt(n); if (r &lt; 128) t += String.fromCharCode(r); else if (r &gt; 127 &amp;&amp; r &lt; 2048) t += String.fromCharCode(r &gt;&gt; 6 | 192), t += String.fromCharCode(63 &amp; r | 128); else if (r &gt;= 55296 &amp;&amp; r &lt;= 56319) { if (n + 1 &lt; e.length) { var i = e.charCodeAt(n + 1); if (i &gt;= 56320 &amp;&amp; i &lt;= 57343) { var a = 1024 * (r - 55296) + (i - 56320) + 65536; t += String.fromCharCode(240 | a &gt;&gt; 18 &amp; 7), t += String.fromCharCode(128 | a &gt;&gt; 12 &amp; 63), t += String.fromCharCode(128 | a &gt;&gt; 6 &amp; 63), t += String.fromCharCode(128 | 63 &amp; a), n++ } } } else t += String.fromCharCode(r &gt;&gt; 12 | 224), t += String.fromCharCode(r &gt;&gt; 6 &amp; 63 | 128), t += String.fromCharCode(63 &amp; r | 128) } return t }; f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 * ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 * 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 * 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a } function md5_1(e) { var t, n, i, a, o, s, m, g, v, y = Array(); for (e = e, y = f, s = 1732584193, m = 4023233417, g = 2562383102, v = 271733878, t = 0; t &lt; y.length; t += 16) n = s, i = m, a = g, o = v, s = l(s, m, g, v, y[t + 0], 7, 3614090360), v = l(v, s, m, g, y[t + 1], 12, 3905402710), g = l(g, v, s, m, y[t + 2], 17, 606105819), m = l(m, g, v, s, y[t + 3], 22, 3250441966), s = l(s, m, g, v, y[t + 4], 7, 4118548399), v = l(v, s, m, g, y[t + 5], 12, 1200080426), g = l(g, v, s, m, y[t + 6], 17, 2821735955), m = l(m, g, v, s, y[t + 7], 22, 4249261313), s = l(s, m, g, v, y[t + 8], 7, 1770035416), v = l(v, s, m, g, y[t + 9], 12, 2336552879), g = l(g, v, s, m, y[t + 10], 17, 4294925233), m = l(m, g, v, s, y[t + 11], 22, 2304563134), s = l(s, m, g, v, y[t + 12], 7, 1804603682), v = l(v, s, m, g, y[t + 13], 12, 4254626195), g = l(g, v, s, m, y[t + 14], 17, 2792965006), m = l(m, g, v, s, y[t + 15], 22, 1236535329), s = c(s, m, g, v, y[t + 1], 5, 4129170786), v = c(v, s, m, g, y[t + 6], 9, 3225465664), g = c(g, v, s, m, y[t + 11], 14, 643717713), m = c(m, g, v, s, y[t + 0], 20, 3921069994), s = c(s, m, g, v, y[t + 5], 5, 3593408605), v = c(v, s, m, g, y[t + 10], 9, 38016083), g = c(g, v, s, m, y[t + 15], 14, 3634488961), m = c(m, g, v, s, y[t + 4], 20, 3889429448), s = c(s, m, g, v, y[t + 9], 5, 568446438), v = c(v, s, m, g, y[t + 14], 9, 3275163606), g = c(g, v, s, m, y[t + 3], 14, 4107603335), m = c(m, g, v, s, y[t + 8], 20, 1163531501), s = c(s, m, g, v, y[t + 13], 5, 2850285829), v = c(v, s, m, g, y[t + 2], 9, 4243563512), g = c(g, v, s, m, y[t + 7], 14, 1735328473), m = c(m, g, v, s, y[t + 12], 20, 2368359562), s = u(s, m, g, v, y[t + 5], 4, 4294588738), v = u(v, s, m, g, y[t + 8], 11, 2272392833), g = u(g, v, s, m, y[t + 11], 16, 1839030562), m = u(m, g, v, s, y[t + 14], 23, 4259657740), s = u(s, m, g, v, y[t + 1], 4, 2763975236), v = u(v, s, m, g, y[t + 4], 11, 1272893353), g = u(g, v, s, m, y[t + 7], 16, 4139469664), m = u(m, g, v, s, y[t + 10], 23, 3200236656), s = u(s, m, g, v, y[t + 13], 4, 681279174), v = u(v, s, m, g, y[t + 0], 11, 3936430074), g = u(g, v, s, m, y[t + 3], 16, 3572445317), m = u(m, g, v, s, y[t + 6], 23, 76029189), s = u(s, m, g, v, y[t + 9], 4, 3654602809), v = u(v, s, m, g, y[t + 12], 11, 3873151461), g = u(g, v, s, m, y[t + 15], 16, 530742520), m = u(m, g, v, s, y[t + 2], 23, 3299628645), s = d(s, m, g, v, y[t + 0], 6, 4096336452), v = d(v, s, m, g, y[t + 7], 10, 1126891415), g = d(g, v, s, m, y[t + 14], 15, 2878612391), m = d(m, g, v, s, y[t + 5], 21, 4237533241), s = d(s, m, g, v, y[t + 12], 6, 1700485571), v = d(v, s, m, g, y[t + 3], 10, 2399980690), g = d(g, v, s, m, y[t + 10], 15, 4293915773), m = d(m, g, v, s, y[t + 1], 21, 2240044497), s = d(s, m, g, v, y[t + 8], 6, 1873313359), v = d(v, s, m, g, y[t + 15], 10, 4264355552), g = d(g, v, s, m, y[t + 6], 15, 2734768916), m = d(m, g, v, s, y[t + 13], 21, 1309151649), s = d(s, m, g, v, y[t + 4], 6, 4149444226), v = d(v, s, m, g, y[t + 11], 10, 3174756917), g = d(g, v, s, m, y[t + 2], 15, 718787259), m = d(m, g, v, s, y[t + 9], 21, 3951481745), s = r(s, n), m = r(m, i), g = r(g, a), v = r(v, o); return (p(s) + p(m) + p(g) + p(v)).toLowerCase() }</code></p>
                  <p>var n = function(e, t) { return e &lt;&lt; t | e &gt;&gt;&gt; 32 - t } , r = function(e, t) { var n, r, i, a, o; return i = 2147483648 &amp; e, a = 2147483648 &amp; t, n = 1073741824 &amp; e, r = 1073741824 &amp; t, o = (1073741823 &amp; e) + (1073741823 &amp; t), n &amp; r ? 2147483648 ^ o ^ i ^ a : n | r ? 1073741824 &amp; o ? 3221225472 ^ o ^ i ^ a : 1073741824 ^ o ^ i ^ a : o ^ i ^ a } , i = function(e, t, n) { return e &amp; t | ~e &amp; n } , a = function(e, t, n) { return e &amp; n | t &amp; ~n } , o = function(e, t, n) { return e ^ t ^ n } , s = function(e, t, n) { return t ^ (e | ~n) } , l = function(e, t, a, o, s, l, c) { return e = r(e, r(r(i(t, a, o), s), c)), r(n(e, l), t) } , c = function(e, t, i, o, s, l, c) { return e = r(e, r(r(a(t, i, o), s), c)), r(n(e, l), t) } , u = function(e, t, i, a, s, l, c) { return e = r(e, r(r(o(t, i, a), s), c)), r(n(e, l), t) } , d = function(e, t, i, a, o, l, c) { return e = r(e, r(r(s(t, i, a), o), c)), r(n(e, l), t) } , f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 <em> ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 </em> 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 <em> 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a } , p = function(e) { var t, n = “”, r = “”; for (t = 0; t &lt;= 3; t++) n += (r = “0” + (e &gt;&gt;&gt; 8 </em> t &amp; 255).toString(16)).substr(r.length - 2, 2); return n }, h = function(e) { e = e.replace(/\x0d\x0a/g, “\n”); for (var t = “”, n = 0; n &lt; e.length; n++) { var r = e.charCodeAt(n); if (r &lt; 128) t += String.fromCharCode(r); else if (r &gt; 127 &amp;&amp; r &lt; 2048) t += String.fromCharCode(r &gt;&gt; 6 | 192), t += String.fromCharCode(63 &amp; r | 128); else if (r &gt;= 55296 &amp;&amp; r &lt;= 56319) { if (n + 1 &lt; e.length) { var i = e.charCodeAt(n + 1); if (i &gt;= 56320 &amp;&amp; i &lt;= 57343) { var a = 1024 <em> (r - 55296) + (i - 56320) + 65536; t += String.fromCharCode(240 | a &gt;&gt; 18 &amp; 7), t += String.fromCharCode(128 | a &gt;&gt; 12 &amp; 63), t += String.fromCharCode(128 | a &gt;&gt; 6 &amp; 63), t += String.fromCharCode(128 | 63 &amp; a), n++ } } } else t += String.fromCharCode(r &gt;&gt; 12 | 224), t += String.fromCharCode(r &gt;&gt; 6 &amp; 63 | 128), t += String.fromCharCode(63 &amp; r | 128) } return t }; f = function(e) { for (var t, n = e.length, r = n + 8, i = 16 </em> ((r - r % 64) / 64 + 1), a = Array(i - 1), o = 0, s = 0; s &lt; n; ) o = s % 4 <em> 8, a[t = (s - s % 4) / 4] = a[t] | e.charCodeAt(s) &lt;&lt; o, s++; return t = (s - s % 4) / 4, o = s % 4 </em> 8, a[t] = a[t] | 128 &lt;&lt; o, a[i - 2] = n &lt;&lt; 3, a[i - 1] = n &gt;&gt;&gt; 29, a }</p>
                  <p>function md5_2(e) { var t, n, i, a, o, s, m, g, v, y = Array(); for (e = e, y = f(e), s = 1732584193, m = 4023233417, g = 2562383102, v = 271733878, t = 0; t &lt; y.length; t += 16) n = s, i = m, a = g, o = v, s = l(s, m, g, v, y[t + 0], 7, 3614090360), v = l(v, s, m, g, y[t + 1], 12, 3905402710), g = l(g, v, s, m, y[t + 2], 17, 606105819), m = l(m, g, v, s, y[t + 3], 22, 3250441966), s = l(s, m, g, v, y[t + 4], 7, 4118548399), v = l(v, s, m, g, y[t + 5], 12, 1200080426), g = l(g, v, s, m, y[t + 6], 17, 2821735955), m = l(m, g, v, s, y[t + 7], 22, 4249261313), s = l(s, m, g, v, y[t + 8], 7, 1770035416), v = l(v, s, m, g, y[t + 9], 12, 2336552879), g = l(g, v, s, m, y[t + 10], 17, 4294925233), m = l(m, g, v, s, y[t + 11], 22, 2304563134), s = l(s, m, g, v, y[t + 12], 7, 1804603682), v = l(v, s, m, g, y[t + 13], 12, 4254626195), g = l(g, v, s, m, y[t + 14], 17, 2792965006), m = l(m, g, v, s, y[t + 15], 22, 1236535329), s = c(s, m, g, v, y[t + 1], 5, 4129170786), v = c(v, s, m, g, y[t + 6], 9, 3225465664), g = c(g, v, s, m, y[t + 11], 14, 643717713), m = c(m, g, v, s, y[t + 0], 20, 3921069994), s = c(s, m, g, v, y[t + 5], 5, 3593408605), v = c(v, s, m, g, y[t + 10], 9, 38016083), g = c(g, v, s, m, y[t + 15], 14, 3634488961), m = c(m, g, v, s, y[t + 4], 20, 3889429448), s = c(s, m, g, v, y[t + 9], 5, 568446438), v = c(v, s, m, g, y[t + 14], 9, 3275163606), g = c(g, v, s, m, y[t + 3], 14, 4107603335), m = c(m, g, v, s, y[t + 8], 20, 1163531501), s = c(s, m, g, v, y[t + 13], 5, 2850285829), v = c(v, s, m, g, y[t + 2], 9, 4243563512), g = c(g, v, s, m, y[t + 7], 14, 1735328473), m = c(m, g, v, s, y[t + 12], 20, 2368359562), s = u(s, m, g, v, y[t + 5], 4, 4294588738), v = u(v, s, m, g, y[t + 8], 11, 2272392833), g = u(g, v, s, m, y[t + 11], 16, 1839030562), m = u(m, g, v, s, y[t + 14], 23, 4259657740), s = u(s, m, g, v, y[t + 1], 4, 2763975236), v = u(v, s, m, g, y[t + 4], 11, 1272893353), g = u(g, v, s, m, y[t + 7], 16, 4139469664), m = u(m, g, v, s, y[t + 10], 23, 3200236656), s = u(s, m, g, v, y[t + 13], 4, 681279174), v = u(v, s, m, g, y[t + 0], 11, 3936430074), g = u(g, v, s, m, y[t + 3], 16, 3572445317), m = u(m, g, v, s, y[t + 6], 23, 76029189), s = u(s, m, g, v, y[t + 9], 4, 3654602809), v = u(v, s, m, g, y[t + 12], 11, 3873151461), g = u(g, v, s, m, y[t + 15], 16, 530742520), m = u(m, g, v, s, y[t + 2], 23, 3299628645), s = d(s, m, g, v, y[t + 0], 6, 4096336452), v = d(v, s, m, g, y[t + 7], 10, 1126891415), g = d(g, v, s, m, y[t + 14], 15, 2878612391), m = d(m, g, v, s, y[t + 5], 21, 4237533241), s = d(s, m, g, v, y[t + 12], 6, 1700485571), v = d(v, s, m, g, y[t + 3], 10, 2399980690), g = d(g, v, s, m, y[t + 10], 15, 4293915773), m = d(m, g, v, s, y[t + 1], 21, 2240044497), s = d(s, m, g, v, y[t + 8], 6, 1873313359), v = d(v, s, m, g, y[t + 15], 10, 4264355552), g = d(g, v, s, m, y[t + 6], 15, 2734768916), m = d(m, g, v, s, y[t + 13], 21, 1309151649), s = d(s, m, g, v, y[t + 4], 6, 4149444226), v = d(v, s, m, g, y[t + 11], 10, 3174756917), g = d(g, v, s, m, y[t + 2], 15, 718787259), m = d(m, g, v, s, y[t + 9], 21, 3951481745), s = r(s, n), m = r(m, i), g = r(g, a), v = r(v, o); return (p(s) + p(m) + p(g) + p(v)).toLowerCase() }</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/小烨" class="author" itemprop="url" rel="index">小烨</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-12 23:39:21" itemprop="dateCreated datePublished" datetime="2020-08-12T23:39:21+08:00">2020-08-12</time>
                </span>
                <span id="/9591.html" class="post-meta-item leancloud_visitors" data-flag-title="某道词典在线翻译JS解密,完全扣js代码解密 （小白基础篇）" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>23k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>21 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9581.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/9581.html" class="post-title-link" itemprop="url">JS 逆向的时候 Python 如何调用 JavaScript 代码？</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="本篇目标"><a href="#本篇目标" class="headerlink" title="本篇目标"></a>本篇目标</h2>
                  <ul>
                    <li>了解为什么我们需要直接调用 JavaScript</li>
                    <li>了解常见的 Python 调用 JavaScript 的库</li>
                    <li>了解一种性能更高的操作方式</li>
                    <li>知道什么场景下应该使用什么方式进行调用</li>
                  </ul>
                  <blockquote>
                    <p>通过本文的学习，在你写爬虫时，你应该会对调用 JavaScript 有一个更清晰的了解，并且你还要了解到一些你平时可能见不到的骚操作。</p>
                  </blockquote>
                  <p>大家如果接触过 JavaScript 逆向的话，应该都知道，通常来说碰到 JS　逆向网站时会有这两种情况：</p>
                  <ul>
                    <li>简单 JS 破解：通过 Python 代码轻松实现</li>
                    <li>复杂的 JS 破解：代码不容易重写，使用程序直接调用 JS 运行获取结果。</li>
                  </ul>
                  <p>对于简单的 JS 来说，我们可以通过 Python 代码，直接重写，轻轻松松的就能搞定。 而对于复的 JS 代码而言呢，由于代码过于复杂，重写太费时费力，且碰到对方更新这就比较麻烦了。所以，我们一般直接使用程序去调用 JS，在 Python 层面就只是获取一个运行结果，这样做相比于重写而言就方便多了。 那么，接下来我带大家看一下两种比较简单的 JS 代码重写。 本文涉及的所有演示代码请到公众号：AI悦创，后台回复：PJS 。来获取即可！</p>
                  <h2 id="1-Base64"><a href="#1-Base64" class="headerlink" title="1. Base64"></a>1. Base64</h2>
                  <p>首先，我们先来看一下 Base64 ，Base64 是我们再写爬虫过程中经常看到的一种编码方式。这边我们来写两个例子。</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">// 原字符</span></span><br><span class="line">NightTeam</span><br><span class="line"><span class="comment">// 编码之后的：</span></span><br><span class="line">TmlnaHRUZWFt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第一个例子如上，是 NightTeam 经过编码是如上面的结果（<code>TmlnaHRUZWFt</code>），如果我们只是通过这个结果来分析的话，它的特征不是很明显。如果是见的不多或者是新手小白的同学，并不会把它往 Base64 方向去想。 然后，我们来看一下第二个例子：</p>
                  <figure class="highlight protobuf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">// 原字符</span></span><br><span class="line">aiyuechuang</span><br><span class="line"><span class="comment">// 编码之后的：</span></span><br><span class="line">YWl5dWVjaHVhbmc=</span><br><span class="line"></span><br><span class="line"><span class="comment">// 原字符</span></span><br><span class="line">Python3</span><br><span class="line">// 编码之后</span><br><span class="line">UHl0aG9uMw==</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第二个例子是 aiyuechuang 编码之后的结果，它的末尾有一个等号，Python3 编码之后末尾有两个等号，这个特征相对第一个就比较明显了。一般我们看到尾号有两个等号时应该大概可以猜到这个就是 Base64 了。 然后，直接解码看一看，如果没有什么特别的话，就可以使用 Python 进行重写了。 同学可以使用以下链接 Base64 编码解码的测试学习：<a href="http://tool.alixixi.com/base64/" target="_blank" rel="noopener">http://tool.alixixi.com/base64/</a> 不过 Base64 也会有一些骚操作，碰到那种情况的时候，我们如果用 Python 重写可能有点麻烦。具体的内容我会在后面的的课程中，单独的跟大家详细的讲解。</p>
                  <h2 id="2-MD5"><a href="#2-MD5" class="headerlink" title="2. MD5"></a>2. MD5</h2>
                  <p>第二个的话就是 MD5 ，MD5 在 Javascript 中并没有标准的库，一般我们都是使用开源库去操作。</p>
                  <blockquote>
                    <p>注意：md5 的话是 <strong>哈希</strong> 并不是加密。</p>
                  </blockquote>
                  <p>下面我来看一个 js 实现 md5 的一个例子： <a href=".3-代码包MD5JS">md5.js</a> 上面的代码时被混淆过的，但是它的主要一个特征还是比较明显的，有一个入口函数：<code>console.log(hex_md5(&quot;aiyuechuang&quot;))</code> 我们可以使用命令行运行一下结果，命令如下：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">node</span> <span class="title">md5</span>.js</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <blockquote>
                    <p>上面的代码自行复制保存为 md5.js 然后运行。</p>
                  </blockquote>
                  <p>运行结果：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ <span class="keyword">node</span> <span class="title">md5</span>.js</span><br><span class="line">e55babec7f5d5cf7bac7872f0481bec1</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们数一下输出的结果的话，会发现这正好 是 32位，通常我们看到 32 位的一个英文数字混合的字符串，应该马上就能想到时 md5 了，这两个操作的话，因为在 Python 中都有对应的库，分别是：Base64 和 hashlib ，大家应该都知道这个我就不多说了。 <strong>例程：Base64 和 hashlib</strong></p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">import</span> <span class="keyword">base64 </span> </span><br><span class="line"><span class="keyword">str1 </span>= <span class="keyword">b'aiyuechuang'</span></span><br><span class="line"><span class="keyword">str2 </span>= <span class="keyword">base64.b64encode(str1)</span></span><br><span class="line"><span class="keyword">print(str2)</span></span><br><span class="line"><span class="keyword">str3 </span>= <span class="keyword">base64.b64decode('YWl5dWVjaHVhbmc=')</span></span><br><span class="line"><span class="keyword">print(str3)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出</p>
                  <figure class="highlight cs">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">b<span class="string">'YWl5dWVjaHVhbmc='</span></span><br><span class="line">b<span class="string">'aiyuechuang'</span></span><br><span class="line">[<span class="meta">Finished in 0.2s</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">import</span> <span class="variable">hashlib</span></span><br><span class="line"></span><br><span class="line"><span class="variable">data</span> = <span class="string">"aiyuechuang"</span></span><br><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">hashlib.md5</span>(<span class="variable">data.encode</span>(<span class="variable">encoding</span> = <span class="string">"UTF-8"</span>)).hexdigest()</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">e55babec7f5d5cf7bac7872f0481bec1</span><br><span class="line">[Finished <span class="keyword">in</span> <span class="number">0.1</span>s]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>像我们前面看到的那些代码，都是比较简单的，他们的算法部分也没有经过修改，所以我们可以使用其他语言和对应的库进行重写。 <strong>但是如果对方把算法部分做了一些改变呢？</strong> 如果代码量比较大也被混淆到看不出特征了，连操作后产生的字符串都看不出，我们就无法直接使用一个现成的库来复写操作了。 而且这种情况下的代码量太大了，直接对着代码重写成 Python 版本也不太现实，对方一更新你就得再重新看一遍，这样显然时非常麻烦的，也非常耗时。 <strong>那么有没有一种更高效的方法呢？</strong> 显然是有的，接下来我们来讲如何通过程序来直接调用 JavaScript 代码，也就是碰到复杂的 JS 时候的处理。</p>
                  <ol>
                    <li>使用 Python 调用 JS</li>
                    <li>一种性能更高的调用方式</li>
                    <li>到底选择哪种方案比较好</li>
                  </ol>
                  <p>首先，我会分享一些使用 Python 调用 JavaScript 的方式，然后会介绍一种性能更高的调用。以及具体使用哪种调用方式以及怎么选择性的使用，最后我会总结一下这些方案存在的小问题。并且会告诉你如何踩坑。</p>
                  <h2 id="3-使用-Python-调用-JS"><a href="#3-使用-Python-调用-JS" class="headerlink" title="3. 使用 Python 调用 JS"></a>3. 使用 Python 调用 JS</h2>
                  <p>我们接下来首先讲一下 Python 中调用 JavaScript。</p>
                  <ul>
                    <li>PyV8</li>
                    <li>Js2Py</li>
                    <li>PyExecJS</li>
                    <li>PyminiRacer</li>
                    <li>Selenium</li>
                    <li>Pyppeteer</li>
                  </ul>
                  <p>Python 调用 JS 库的话，光是我了解的话，目前就有这么一堆，接下来我们就来依次来介绍这些库。</p>
                  <h3 id="3-1-PyV8"><a href="#3-1-PyV8" class="headerlink" title="3.1 PyV8"></a>3.1 PyV8</h3>
                  <ul>
                    <li>V8 是谷歌开源的 JavaScript 引擎，被使用在了 Chrome 中</li>
                    <li>PyV8 是 V8 引擎的一个 Python 层的包装，可以用来调用 V8 引擎执行 JS 代码</li>
                    <li>网上有很多使用它来执行 JS 代码的文章</li>
                    <li>年久失修，最新版本是 2010年的（<a href="https://pypi.org/project/PyV8/#history" target="_blank" rel="noopener">https://pypi.org/project/PyV8/#history</a>)</li>
                    <li>存在内存泄漏问题，所以不建议使用</li>
                  </ul>
                  <p>首先来看一下什么是 PyV8，V8 是谷歌开源的 JavaScript 引擎，被使用在了 Chrome 浏览器中，后来因为有人想在 Python 上调用它（V8），于是就有了 PyV8。 那 PyV8 实际上是 V8 引擎的一个 Python 层的包装，可以用来调用 V8 引擎执行 JS 代码，但是这个我不推荐使用它，那我既然不推荐大家使用，我为什么又要讲它呢？ <strong>其实，是这样的：</strong> 虽然目前网上有很多文章使用它执行 JS 代码，但是这个 PyV8 实际上已经年久失修了，而且它最新的一个正式版本还是 2010年的，可见是有多久远了，链接在上方可以执行访问查看。而且，如果你实际使用过的话，你应该会发现它存在一些<strong>内存泄漏</strong>的问题。 所以，这边我拿出来说一下，避免有人踩坑。接下来我们来说一下第二个 JS2Py。</p>
                  <h3 id="3-2-Js2Py"><a href="#3-2-Js2Py" class="headerlink" title="3.2 Js2Py"></a>3.2 Js2Py</h3>
                  <ul>
                    <li>Js2Py 是一个纯 Python 实现的 JavaScript 解释器和翻译器</li>
                    <li>虽然 2019年依然有更新，但那也是 6月份的事情了，而且它的 issues 里面有很多的 bug 没有修复（<a href="https://github.com/PiotrDabkowski/Js2Py/issues" target="_blank" rel="noopener">https://github.com/PiotrDabkowski/Js2Py/issues</a>）。</li>
                  </ul>
                  <p>Js2Py 是一个纯 Python 实现的 JavaScript 解释器和翻译器，它和 PyV8 一样，也是有挺多文章提到这个库，然后来调用 JS 代码。 但是，Js2Py 虽然在2019年仍然更新，但那也是 6月份的事情了，而且它的 issues 里面有很多的 bug 没有修复（<a href="https://github.com/PiotrDabkowski/Js2Py/issues" target="_blank" rel="noopener">https://github.com/PiotrDabkowski/Js2Py/issues</a>）。另外，Js2Py 本身也存在一些问题，就解释器部分来说： <strong>解释器部分：</strong></p>
                  <ul>
                    <li>性能不高</li>
                    <li>存在一些 BUG</li>
                  </ul>
                  <p>那不仅仅就解释器部分，还有翻译器部分：</p>
                  <ul>
                    <li>对于高度混淆的大型 JS 会转换失败</li>
                    <li>而且转换出来的代码可读性差、性能不高</li>
                  </ul>
                  <p>总之来讲，它在各个方面来说都不太适合我们的工作场景，所以也是不建议大家使用的。</p>
                  <h3 id="3-3-PyMinRacer"><a href="#3-3-PyMinRacer" class="headerlink" title="3.3 PyMinRacer"></a>3.3 PyMinRacer</h3>
                  <ul>
                    <li>同样是 V8 引擎的包装，和 PyV8 的效果一样</li>
                    <li>一个继任 PyExecJS 和 PyramidV8 的库</li>
                    <li>一个比较新的库</li>
                  </ul>
                  <p>这个库也是一个 PyV8 引擎包装，它的效果和 PyV8 的效果一样的。 而且作者号称这是一个继任 PyExecJS 和 PyramidV8 的库，乍眼一看挺唬人的，不过由于它是一个比较新的库，我这边就没有过多的尝试了，也没有再实际生产环境中使用过，所以不太清楚会有什么坑，感兴趣的朋友，大家可以自己去尝试一下。</p>
                  <h3 id="3-4-PyExecJS"><a href="#3-4-PyExecJS" class="headerlink" title="3.4 PyExecJS"></a>3.4 PyExecJS</h3>
                  <ul>
                    <li>一个最开始诞生于 Ruby 中的库，后来被移植到了 Python 上</li>
                    <li>较新的文章一般都会说用它来执行 JS 代码</li>
                    <li>有多个引擎可选，但一般我们会选择使用 NodeJS 作为引擎来执行代码</li>
                  </ul>
                  <p>接下来我要说的是 PyExecJS ，这个库一个最开始诞生于 Ruby 中的库，后来人被移植到了 Python 上，目前看到一些比较新的文章都是用它来执行 JS 代码的，然后它是有多个引擎可以选择的，我们一般选择 <strong>NodeJS</strong> 作为它的一个引擎执行代码，毕竟 <strong>NodeJS</strong> 的速度是比较快的而且配置起来比较简单，那我带大家来看一下 PyExecjs 的使用。</p>
                  <h4 id="3-4-5-PyExecJS-的使用"><a href="#3-4-5-PyExecJS-的使用" class="headerlink" title="3.4.5 PyExecJS 的使用"></a>3.4.5 PyExecJS 的使用</h4>
                  <ol>
                    <li>安装 JS 运行环境这里推荐安装 Node.js，安装方便，执行效率也高。</li>
                  </ol>
                  <p>首先我们就是要安装引擎了，这个引擎指的就是 JS 的一个运行环境，这边推荐使用 Node.js。</p>
                  <blockquote>
                    <p>注意：虽然 Windows 上有个系统自带的 JScript，可以用来作为 PyExecjs 的引擎，但是这个 JScript 很容易与其他的引擎有一个不一样的地方，容易踩到一些奇奇怪怪的坑。所以请大家务必要安装一个其他的引擎。比如说我们这里安装 Node.js 。</p>
                  </blockquote>
                  <p>那上面装完 Nodejs 之后呢，我们就需要执行安装 PyExecjs 了：</p>
                  <ol>
                    <li>
                      <p>安装 PyExecJS</p>
                      <figure class="highlight cmake">
                        <table>
                          <tr>
                            <td class="gutter">
                              <pre><span class="line">1</span><br></pre>
                            </td>
                            <td class="code">
                              <pre><span class="line">pip <span class="keyword">install</span> pyexecjs</span><br></pre>
                            </td>
                          </tr>
                        </table>
                      </figure>
                    </li>
                  </ol>
                  <p>这边我们使用上面的 pip 就可以进行安装了。 那么我们现在环境就准备好了，可以开始运行了。</p>
                  <ol>
                    <li>代码示例（检测运行环境）</li>
                  </ol>
                  <p>首先，我们打开 IPython 终端，执行一下一下两行代码，以下也给出了运行结果：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">In</span> [<span class="number">1</span>]: import execjs</span><br><span class="line"></span><br><span class="line"><span class="keyword">In</span> [<span class="number">2</span>]: execjs.get().name <span class="comment"># 查看调用环境</span></span><br><span class="line">Out[<span class="number">2</span>]: '<span class="keyword">Node</span>.<span class="title">js</span> (V8)'</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><code>execjs.get() # 查看调用的环境</code>用此来看看我们的库能不能检测到 nodejs，如果不能的话那就需要手动设置一下，不过一般像我上面一样正常输出 <code>node.js</code> 就可以了。 如果，你检测出来的引擎不是 <code>node.js</code> 的话，那你就需要手动设置一下了，这里有两种设置形式，我在下方给你写出来了： <strong>选择不同引擎进行解析</strong></p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># 长期使用</span></span><br><span class="line">os.environ[<span class="string">"EXECJS_RUNTIME"</span>]=<span class="string">"Node"</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 临时使用</span></span><br><span class="line">import execjs.runtime_names</span><br><span class="line"><span class="keyword">node</span><span class="title">=execjs</span>.get(execjs.runtime_names.<span class="keyword">Node</span><span class="title">)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>由上边可知，我们有两种形式：一种是长期使用的，通过环境变量的形式，通过把环境变量改成大写的 EXECJS_RUNTIME 然后将其值赋值为 Node。 另一种的话，将它改成临时使用的一种方式，这种是直接使用 get，这种做法的话，你在使用的时候就需要使用 node 变量了，不能直接导入 PyExecjs 来直接开始使用，相对麻烦一些。 接下来，就让我们正式使用 PyExecJS 这个包吧。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">In [<span class="number">8</span>]: <span class="keyword">import</span> execjs</span><br><span class="line"></span><br><span class="line">In [<span class="number">9</span>]: e = execjs.eval(<span class="string">'a = new Array(1, 2, 3)'</span>) # 可以直接执行 JS 代码</span><br><span class="line"></span><br><span class="line">In [<span class="number">10</span>]: print(e)</span><br><span class="line">[<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>PyExecjs 最简单的用法就是导入包，然后通过 eval 这个方法并传入简单的 JS 代码来执行。但是我们正常情况下肯定不会这么使用，因为我们的 JS 代码是比较复杂的而且 JS 代码内容也是比较多的。</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @Author: clela</span></span><br><span class="line"><span class="comment"># @Date:   2020-03-24 13:54:27</span></span><br><span class="line"><span class="comment"># @Last Modified by:   aiyuechuang</span></span><br><span class="line"><span class="comment"># @Last Modified time: 2020-04-03 08:44:15</span></span><br><span class="line"><span class="comment"># @公众号：AI悦创</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">12</span>]: <span class="keyword">import</span> execjs</span><br><span class="line"></span><br><span class="line">In [<span class="number">13</span>]: jstext = <span class="string">"""</span></span><br><span class="line"><span class="string">    ...: function hello(str)&#123;return str;&#125;</span></span><br><span class="line"><span class="string">    ...: """</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">14</span>]: ctx = execjs.compile(jstext) <span class="comment"># 编译 JS 代码</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">15</span>]: a = ctx.call(<span class="string">"hello"</span>, <span class="string">"hello aiyc"</span>)</span><br><span class="line"></span><br><span class="line">In [<span class="number">16</span>]: print(a)</span><br><span class="line">hello aiyc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样的话，我们一般通过使用第二种方式，第二种方式是通过使用 compile 对 JS 字符串进行编译，这个编译操作其实就是把参数（jstext）里面的那段 JS 代码给放到一个叫 Context 的上下文中，它并不是我们平时编译程序所说的编译。然后我们 调用 call 方法进行执行。 第一个参数是我们调用 JS 中的的函数名，也就是 hello。然后后面跟着的 hello aiyc 就是参数，也就是我们 JS 中需要传入到 str 的参数。如果 JS 中存在多个参数，我们就直接在后面打个逗号，然后接着写下一个参数就好了。 接下来我们来看一个具体的代码： <a href=".3-代码包PyExecjsDemo">aes_demo.js</a> 这边我准备了一个 CryptoJS 的一个 JS 文件，CryptoJS 它是一个包含各种加密哈希编码算法的一个开源库，很多网站都会用它提供的函数来生成参数，那么这边我是写了如上面这样的代码，用来调用它里面的 AES 加密参数，来加密一下我提供的字符串。</p>
                  <blockquote>
                    <p><strong>注意</strong>：JS 代码不要放在和 Python 代码同一个文件中，尽量放在单独的 js 文件中，因为我们的 JS 文件内容比较多。然后通过读取文件的方式，</p>
                  </blockquote>
                  <p><a href=".3-代码包PyExecjsDemo">run_aes.py</a></p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># Python 文件：run_aes.py</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @时间 : 2020-04-06 00:00</span></span><br><span class="line"><span class="comment"># @作者 : AI悦创</span></span><br><span class="line"><span class="comment"># @文件名 : run_aes.py</span></span><br><span class="line"><span class="comment"># @公众号: AI悦创</span></span><br><span class="line"><span class="keyword">from</span> pprint import pprint</span><br><span class="line"></span><br><span class="line">import execjs</span><br><span class="line">import pathlib</span><br><span class="line">import os</span><br><span class="line"></span><br><span class="line">js_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))</span><br><span class="line">js_path = js_path / <span class="string">"crypto.js"</span></span><br><span class="line">with js_path.open(<span class="string">'r'</span>, <span class="attribute">encoding</span>=<span class="string">"utf-8"</span>) as f:</span><br><span class="line">   <span class="built_in"> script </span>= f.read()</span><br><span class="line"></span><br><span class="line">c = <span class="string">"1234"</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 传入python中的变量</span></span><br><span class="line"><span class="builtin-name">add</span> = (<span class="string">''</span><span class="string">'</span></span><br><span class="line"><span class="string">aesEncrypt = function() &#123;</span></span><br><span class="line"><span class="string">    result=&#123;&#125;</span></span><br><span class="line"><span class="string">    var t = CryptoJS.MD5("login.xxx.com"),</span></span><br><span class="line"><span class="string">        i = CryptoJS.enc.Utf8.parse(t),</span></span><br><span class="line"><span class="string">        r = CryptoJS.enc.Utf8.parse("1234567812345678"),</span></span><br><span class="line"><span class="string">        u = CryptoJS.AES.encrypt('</span><span class="string">''</span> + <span class="string">"'&#123;&#125;'"</span>.format(c) + <span class="string">''</span><span class="string">',i, &#123;</span></span><br><span class="line"><span class="string">        iv: r</span></span><br><span class="line"><span class="string">    &#125;);</span></span><br><span class="line"><span class="string">        result.t=t.toString()</span></span><br><span class="line"><span class="string">        result.i =i.toString()</span></span><br><span class="line"><span class="string">        result.r =r.toString()</span></span><br><span class="line"><span class="string">        result.u =u.toString()</span></span><br><span class="line"><span class="string">        return result</span></span><br><span class="line"><span class="string">    &#125;;</span></span><br><span class="line"><span class="string">    '</span><span class="string">''</span>)</span><br><span class="line">script =<span class="built_in"> script </span>+ add</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">"script"</span>,script)</span><br><span class="line"></span><br><span class="line">x = execjs.compile(script)</span><br><span class="line">result = x.call(<span class="string">"aesEncrypt"</span>)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我通过读取文件的方式，将 js 文件读取进来，把代码读取到我们的字符串里面，这样一方面方便我们管理，另一方面也可以直接通过代码检测自动补全功能，使用起来会比较方便。 然后，这里我们有一个小技巧，我们可以通过 format 字符串拼接的形式，将 Python 中的变量，也就是上面的变量 <code>c</code> 然后将这个变量写入到 Js 代码中，从而变相的实现了通过调用 JS 函数，在没有参数的情况下修改 JS 代码中的特定变量的值。最后我们拼接好了我我们的 JS 代码（add 和 script）。 拼完 JS 代码之后，我们这边再常规的进行一个操作，调用 Call 方法执行 aesEncrypt 这样一个函数，需要注意的是，这个代码里面 return 出来的 JS，它是一个 object，JS 中的 object 也就是 Python 中的<strong>字典</strong>。 <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200812133011.png" alt=""> 我们实际使用时，如果需要在 Python 中拿到 object 的话，建议把它转换成一个 json 字符串，而不是直接的把结果 return 出来。 因为，有些时候 PyExecjs 对 object 的转换会出现问题，所以我们可能会拿到一些类似于将字典直接用 str 函数包裹后转为字符串的一个东西，这样的话它是无法通过正常的方式去解析的。 或者说你也可能会遇到其情况的报错，总之大家最好先转一下 json 字符串，然后再 return 避免踩坑。这是我们的一个代码。 <strong>接下来我们来说一下，PyExecJS 存在的一些问题主要有以下两点：</strong></p>
                  <ul>
                    <li>执行大型 JS 时会有点慢（这个是因为，每次执行 JS 代码的时候，都是从命令行去调用到的 JS，所以 JS 代码越复杂的话，nodejs 的初始化时间就越长，这个基本上是无解的）</li>
                    <li>特殊编码的输入或输出参数会出现报错的情况（因为，是从命令行调用的，所以在碰到一些特殊字符输入或输出参数或者 JS 代码本身就有一些特殊字符的情况下，就会直接执行不了，给你抛出一个异常。不过这个跟系统的命令行默认编码有一定关系，具体的话这里就不深究了，直接就说解决方案吧。）</li>
                    <li>可以把输入或输出的参数使用 Base64 编码一下（如果看报错是 JS 代码部分导致的，那就去看看能不能删除代码中的那部分字符或者你自己 new 一个上下文对象，将那个名叫 tempfile 的参数打开，这样在调用的时候，它就直接去执行那个文件了，不过大量调用的情况下，可能会对磁盘造成一定压力。</li>
                  </ul>
                  <p>而如果参数不充分导致的话，有个很简单的方法：就是把参数使用 Base64 编码一下，因为编码之后出来的字符串，我们知道 Base64 编码之后是生成英文和数字组成的。这样就没有特殊符号了。所以就不会出现问题了。） 关于 PyExecejs 的相关东西就介绍到这里了，我们来看一些其他的内容。</p>
                  <h3 id="3-5-其他使用-Python-调用-JS-的骚操作"><a href="#3-5-其他使用-Python-调用-JS-的骚操作" class="headerlink" title="3.5 其他使用 Python 调用 JS 的骚操作"></a>3.5 其他使用 Python 调用 JS 的骚操作</h3>
                  <p>前面说的都是非浏览器环境下直接调用 JS 的操作，但是还有一些市面上根本没人提到的骚操作，其实也挺好用的，接下来我给大家介绍一下：</p>
                  <ol>
                    <li>Selenium</li>
                  </ol>
                  <ul>
                    <li>一个 web 自动化测试框架，可以驱动各种浏览器进行模拟人工操作</li>
                    <li>用于渲染页面以方便提取数据或过验证码</li>
                    <li>也可以直接驱动浏览器执行 JS</li>
                  </ul>
                  <p>这个大家是比较熟悉的，它是一个外部自动化的测试框架，可以驱动各种浏览器进行模拟人工操作，很多文章或者培训班的课程，都会提到它在爬虫方面的一个使用，比如用它采集一些动态页面，或者用来过一些滑动验证码之类的。 不过我们这里不用它来做这些事，我们要做的是用它来执行 JS 代码，因为这样的话是直接在浏览器环境下执行的 ，所以的话它是省了很多事，那么 Selenium 执行 JS 的核心代码，实际上就下面一行：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">js</span> = <span class="string">"一大段 JS"</span></span><br><span class="line"><span class="attr">result</span> = browser.execute_script(js)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们来看一下实际的例子： <a href=".3-代码包SeleniumDemo">SeleniumDemo</a> 进入项目根目录，输入：<code>python server.py</code></p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python <span class="keyword">server</span>.py</span><br><span class="line"> * Serving Flask app "server" (lazy loading)</span><br><span class="line"> * Environment: production</span><br><span class="line">   <span class="built_in">WARNING</span>: This <span class="keyword">is</span> a development <span class="keyword">server</span>. <span class="keyword">Do</span> <span class="keyword">not</span> use it <span class="keyword">in</span> a production deployment.</span><br><span class="line">   Use a production WSGI <span class="keyword">server</span> <span class="keyword">instead</span>.</span><br><span class="line"> * <span class="keyword">Debug</span> mode: <span class="keyword">on</span></span><br><span class="line"> * Restarting <span class="keyword">with</span> stat</span><br><span class="line"> * Debugger <span class="keyword">is</span> active!</span><br><span class="line"> * Debugger PIN: <span class="number">262</span><span class="number">-966</span><span class="number">-819</span></span><br><span class="line"> * Running <span class="keyword">on</span> http://<span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span>:<span class="number">5002</span>/ (Press CTRL+C <span class="keyword">to</span> quit)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>访问 <code>localhost:5002</code> 我们进入网页之后，有这样的一句话： <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200812133024.png" alt=""> <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200812133028.png" alt=""></p>
                  <hr>
                  <p>每次刷新都会显示不同的内容，查看源代码的话，会发现这个页面中的源代码里面没有对应页面显示的那句话，而是只有一个 input 标签。 <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200812133032.png" alt=""> 我还能观察到，input 标签里面有两个属性，一个是 id、一个是 data，这两个是比较关键的属性，然后我们还发现这里面引用了一个 js 文件，所以这个网页最终结果实际上是通过 JS 文件，然后一系列的操作生成的，那接下来我就来看看 JS 文件，做了什么工作。 <a href=".3-代码包SeleniumDemostaticjs">js</a> 我们可以看见，这个 JS 文件最后一句，有一个 <code>window.onload =doit</code> 的这样一代码，这个我们知道，当页面加载完成之后，立即执行这个 JS 方法。</p>
                  <figure class="highlight javascript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">doit</span>(<span class="params"></span>) </span>&#123;</span><br><span class="line">    <span class="keyword">let</span> browser_type=BrowserType();</span><br><span class="line">    <span class="built_in">console</span>.log(browser_type)</span><br><span class="line">    <span class="keyword">let</span> supporter =browser_type.supporter</span><br><span class="line">    <span class="keyword">if</span>(supporter===<span class="string">"chrome"</span>)&#123;</span><br><span class="line">        Base64.run(<span class="string">'base64'</span>, <span class="string">'data'</span>,supporter)</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后这个方法里面做了一个这样一个操作：<code>let browser_type=BrowserType();</code> 首先去判断 <code>supporter</code> 是否等于 <code>Chrome</code> 这个 <code>supporter</code> 实际上有一个 <code>browser_type</code> 这个 <code>browser_type</code> 实际上就是检测浏览器等一系列参数，然后我们获取它里面的 <code>supporter</code> 属性，当 <code>supporter</code>（ <code>supporter =browser_type.supporter</code> ）等于 Chrome 的时候，我们再去执行这个 run 函数。</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">run: <span class="keyword">function</span> (id, attr,supporter) &#123;</span><br><span class="line">        <span class="keyword">let</span> all_str = <span class="constructor">$(<span class="params">id</span>)</span>.get<span class="constructor">Attribute(<span class="params">attr</span>)</span></span><br><span class="line">        <span class="keyword">let</span> end_index=supporter.length+<span class="number">58</span></span><br><span class="line">        <span class="module-access"><span class="module"><span class="identifier">Base64</span>.</span><span class="module"><span class="identifier">_keyStr</span> </span></span>= all_str.substring(<span class="number">0</span>, end_index)</span><br><span class="line">        <span class="keyword">let</span> charset = all_str.substring(<span class="number">64</span>, all_str.length)</span><br><span class="line">        <span class="keyword">let</span> encoded = <span class="module-access"><span class="module"><span class="identifier">Base64</span>.</span></span>decode(charset,supporter);</span><br><span class="line">        <span class="constructor">$(<span class="params">id</span>)</span>.value = encoded;</span><br><span class="line">    &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>也就是 run 函数里面做了一系列操作，然后我传入的 id 可以通过看一下上面的函数 <code>doit</code> 可知传入的是 <code>Base64</code> 也就是说，实际上对 input 这个标签做了一个取值的操作，然后到这边我们就这整体一个过程将会用 JS 去模拟，所以这边我就不细说了。 最终会把这样的一个结果去通过 input.value 属性把值复制到 input 中，也就是我们最终看到的那样一个结果，到目前我就把这个 js 大概做了一件什么样的事情就已经讲的差不多了。接下来我们去看一下 Selenium 这边。 代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @Time : 2020-04-01 20:56</span></span><br><span class="line"><span class="comment"># @Author : aiyuehcuang</span></span><br><span class="line"><span class="comment"># @File : demo.py</span></span><br><span class="line"><span class="comment"># @Software: PyCharm</span></span><br><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.by <span class="keyword">import</span> By</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support.ui <span class="keyword">import</span> WebDriverWait</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support <span class="keyword">import</span> expected_conditions <span class="keyword">as</span> EC</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.action_chains <span class="keyword">import</span> ActionChains</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.keys <span class="keyword">import</span> Keys</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.chrome.options <span class="keyword">import</span> Options</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_text</span><span class="params">(id,attr)</span>:</span></span><br><span class="line">    <span class="comment">###  拼接字符串注意&#123;&#125;要写出&#123;&#123;&#125;&#125;</span></span><br><span class="line">    script=(<span class="string">"""</span></span><br><span class="line"><span class="string">       let bt=BrowserType();</span></span><br><span class="line"><span class="string">       let id='&#123;id&#125;';</span></span><br><span class="line"><span class="string">       let attr='&#123;attr&#125;';</span></span><br><span class="line"><span class="string">       let supporter =bt.supporter;</span></span><br><span class="line"><span class="string">    const run=function()&#123;&#123;</span></span><br><span class="line"><span class="string">    let all_str = $(id).getAttribute(attr)</span></span><br><span class="line"><span class="string">    let end_index=supporter.length+58</span></span><br><span class="line"><span class="string">    Base64._keyStr = all_str.substring(0, end_index)</span></span><br><span class="line"><span class="string">    let charset = all_str.substring(64, all_str.length)</span></span><br><span class="line"><span class="string">    let encoded = Base64.decode(charset,supporter);</span></span><br><span class="line"><span class="string">    return encoded</span></span><br><span class="line"><span class="string">&#125;&#125;</span></span><br><span class="line"><span class="string">    return run()</span></span><br><span class="line"><span class="string">    """</span>).format(id=id,attr=attr)</span><br><span class="line">    <span class="keyword">return</span> script</span><br><span class="line"></span><br><span class="line">chrome_option = Options()</span><br><span class="line">chrome_option.add_argument(<span class="string">"--headless"</span>)</span><br><span class="line">chrome_option.add_argument(<span class="string">"--disable-gpu"</span>)</span><br><span class="line">chrome_option.add_argument(<span class="string">'--ignore-certificate-errors'</span>)  <span class="comment"># SSL保存</span></span><br><span class="line">browser = webdriver.Chrome(options=chrome_option)</span><br><span class="line">wait = WebDriverWait(browser, <span class="number">10</span>)</span><br><span class="line"><span class="comment"># 启动浏览器，获取网页源代码</span></span><br><span class="line">mainUrl = <span class="string">"http://127.0.0.1:5002/"</span></span><br><span class="line">browser.get(mainUrl)</span><br><span class="line">result=browser.execute_script(get_text(<span class="string">"base64"</span>,<span class="string">"data"</span>))</span><br><span class="line">print(result)</span><br><span class="line">time.sleep(<span class="number">10</span>)</span><br><span class="line">browser.quit()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这边关键的一行代码是：通过 <code>execute_script(get_text(&quot;base64&quot;,&quot;data&quot;))</code> 这样的一句话去执行这个函数，这个函数实际上就是返回一段 JS 代码，这边实际上就是去模拟构造 run 所需要的一些参数，然后把最终的结果返回回去。 这里有两点需要注意：</p>
                  <ol>
                    <li>如果里面存在拼接字符串的时候，注意花括号实际上要写两个</li>
                    <li>如果需要在后面需要获取 JS 返回的值，所以我们上面的代码需要加上 return 来返回 run 函数的结果</li>
                  </ol>
                  <p>我们可以运行一下代码，输出结果如下：</p>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">$ </span>python demo.py</span><br><span class="line"></span><br><span class="line">DevTools listening on <span class="symbol">ws:</span>/<span class="regexp">/127.0.0.1:59507/devtools</span><span class="regexp">/browser/edbe</span>51d8<span class="number">-744</span>d<span class="number">-447</span>d<span class="number">-9304</span>-e9551a2a6421</span><br><span class="line">[0407/<span class="number">184920.601</span><span class="symbol">:INFO</span><span class="symbol">:CONSOLE</span>(<span class="number">286</span>)] <span class="string">"[object Object]"</span>, <span class="symbol">source:</span> <span class="symbol">http:</span>/<span class="regexp">/127.0.0.1:5002/static</span><span class="regexp">/js/base</span>64.js (<span class="number">286</span>)</span><br><span class="line">生活不是等待暴风雨过去，而是要学会在雨中跳舞。</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们可以看到，我们程够获取到了结果。 这个例子因为它用到了检测浏览器的属性，而且它检测完属性之后会把属性值一直往下传，我们可以从上面的代码中看到它有很多地方使用。 所以，如果我们用 PyExecjs 来写的话，就需要修改很多参数，这样就很不方便了。因为我们需要去模拟这些浏览器参数，我这边写的例子比较简单，像那种更加复杂的。像获取更多的浏览器的一个属性的话，用 PyExecjs 再去写的时候，可能没有浏览器这样的一个环境，所以 PyExecjs 没有 Selenium 有优势。 当然，除了 Selenium 以为，还有一个叫做 Pyppeteer 的库，也是比较常见。 为了控制文章篇幅，咱们下次再续咯，记得关注公众号：AI悦创！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/黄家宝|AI悦创" class="author" itemprop="url" rel="index">黄家宝|AI悦创</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-12 15:34:22" itemprop="dateCreated datePublished" datetime="2020-08-12T15:34:22+08:00">2020-08-12</time>
                </span>
                <span id="/9581.html" class="post-meta-item leancloud_visitors" data-flag-title="JS 逆向的时候 Python 如何调用 JavaScript 代码？" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>12k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>11 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9567.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9567.html" class="post-title-link" itemprop="url">qimingpian数据接口分析</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="qimingpian接口加密分析"><a href="#qimingpian接口加密分析" class="headerlink" title="qimingpian接口加密分析"></a>qimingpian接口加密分析</h1>
                  <p>工具：Chrome + NodeJS + Pycharm <a href="https://github.com/Payne-Wu/PythonScrape/tree/master/JS%E5%88%86%E6%9E%90/%E5%90%AF%E6%98%8E%E7%89%87" target="_blank" rel="noopener">点击获取结果</a> 如果能留下小星星就最好啦</p>
                  <h2 id="抓包"><a href="#抓包" class="headerlink" title="抓包"></a>抓包</h2>
                  <p>调出开发者工具，直接到xhr(这里点击改变的时候并未发生网址变更、所以这是Ajax)</p>
                  <h2 id="参数寻找"><a href="#参数寻找" class="headerlink" title="参数寻找"></a>参数寻找</h2>
                  <p>一共就两个包，但Preview里面没有数据，but几KB的包没有鬼？反正我不相信<img src="Note网站JS解密与逆向qimingpian1.参数寻找.png" alt="1.参数寻找"> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/1.参数寻找.png" alt=""></p>
                  <h3 id="追根揭底"><a href="#追根揭底" class="headerlink" title="追根揭底"></a>追根揭底</h3>
                  <p>直接把encrypt_data，拉出来全局搜索（ctrl + shift + F），encrypt_data参数一共六个，但就只有这一个最可疑（我就是不告诉你为什么。。。），其实你看看周围的函数你就会发现，TmD一个个返回啥呀，不是错误就是上传失败。封IP的信息就放了。怕了怕了 <img src="Note网站JS解密与逆向qimingpian2.有猫腻.png" alt="2.有猫腻"> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/2.有猫腻.png" alt=""> 在console里面打印一下Object(u.a)(e.encrypt_data) 初一看，好像是又好像不是（仅有部分信息）</p>
                  <blockquote>
                    <p>只有标题，为什么没信息呢？ 我告诉你为什么，因为数据被加密了，只给你看标题，充钱就给你看。 不慌，不慌。那个xx说过我离成功就一步了</p>
                  </blockquote>
                  <p>点击 下一步<img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/next.png" alt="">没错，就是它。老板，求解密一下？ ok，感谢老板。再次在console里面打印一下Object(u.a)(e.encrypt_data) 当当当~ <img src="Note网站JS解密与逆向qimingpian2.3有猫腻-揭开神秘面纱.png" alt="2.3有猫腻-揭开神秘面纱"> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/2.3有猫腻-揭开神秘面纱.png" alt=""> ok，那它是怎么来的呢？</p>
                  <blockquote>
                    <p>都晓得它是这里解密出来的，还不就进去搞他呗</p>
                  </blockquote>
                  <p><img src="Note网站JS解密与逆向qimingpian3.紧随其后.png" alt="3.紧随其后"> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/3.紧随其后.png" alt=""> 当当当~，扣它，把这个函数扣出来（快到我怀里来~） <img src="Note网站JS解密与逆向qimingpian3.紧随其后-加密参数.png" alt="3.紧随其后-加密参数"> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/3.紧随其后-加密参数.png" alt=""> 到这里就基本上把主函数弄完了，但是还没有完 a.a.decode(t)这个鬼我们还不晓得，进去找他，扣它 <img src="Note网站JS解密与逆向qimingpian4.缺啥补啥.png" alt="4.缺啥补啥"><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/4.缺啥补啥.png" alt=""></p>
                  <figure class="highlight matlab">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">decode = <span class="function"><span class="keyword">function</span> <span class="params">(t)</span> &#123;</span></span><br><span class="line">    var e = (t = String(t).replace(f, <span class="string">""</span>)).<span class="built_in">length</span>;</span><br><span class="line">    e <span class="comment">% 4 == 0 &amp;&amp; (e = (t = t.replace(/==?$/, "")).length),</span></span><br><span class="line">    (e <span class="comment">% 4 == 1 || /[^+a-zA-Z0-9/]/.test(t)) &amp;&amp; l("Invalid character: the string to be decoded is not correctly encoded.");</span></span><br><span class="line">    <span class="keyword">for</span> (var n, r, <span class="built_in">i</span> = <span class="number">0</span>, o = <span class="string">""</span>, a = <span class="number">-1</span>; ++a &lt; e;)</span><br><span class="line">        r = c.indexOf(t.charAt(a)),</span><br><span class="line">            n = <span class="built_in">i</span> <span class="comment">% 4 ? 64 * n + r : r,</span></span><br><span class="line">        <span class="built_in">i</span>++ <span class="comment">% 4 &amp;&amp; (o += String.fromCharCode(255 &amp; n &gt;&gt; (-2 * i &amp; 6)));</span></span><br><span class="line">    <span class="keyword">return</span> o</span><br><span class="line">&#125;,</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">o</span><span class="params">(t)</span> &#123;</span></span><br><span class="line">    <span class="keyword">return</span> JSON.parse(s(<span class="string">"5e5062e82f15fe4ca9d24bc5"</span>, a.a.decode(t), <span class="number">0</span>, <span class="number">0</span>, <span class="string">"012345677890123"</span>, <span class="number">1</span>))</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的参数t，还不晓得，既然是外面传进来的，那么它要么是js生成的，要么就是全局的。全前面找，去console里面测一下，测多次。如果是不变的那么它就是一个全局参数。 拿过来就好，然后在console里面copy（t）。 同理，参数c，和f也是 但是c， 和f 就在decode函数前面，拿了就好 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/4.over_.png" alt=""> 完成！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-09 22:47:52" itemprop="dateCreated datePublished" datetime="2020-08-09T22:47:52+08:00">2020-08-09</time>
                </span>
                <span id="/9567.html" class="post-meta-item leancloud_visitors" data-flag-title="qimingpian数据接口分析" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9549.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> JavaScript <i class="label-arrow"></i>
                  </a>
                  <a href="/9549.html" class="post-title-link" itemprop="url">MiGu登录参数分析</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="MiGu登录参数分析"><a href="#MiGu登录参数分析" class="headerlink" title="MiGu登录参数分析"></a>MiGu登录参数分析</h1>
                  <h1 id="目标：分析咪咕视频登录参数（enpassword、fingerPrint、fingerPrintDetail）"><a href="#目标：分析咪咕视频登录参数（enpassword、fingerPrint、fingerPrintDetail）" class="headerlink" title="目标：分析咪咕视频登录参数（enpassword、fingerPrint、fingerPrintDetail）"></a>目标：分析咪咕视频登录参数（<strong>enpassword</strong>、<strong>fingerPrint</strong>、<strong>fingerPrintDetail</strong>）</h1>
                  <h2 id="工具-NodeJs-Chrome-开发者工具"><a href="#工具-NodeJs-Chrome-开发者工具" class="headerlink" title="工具:NodeJs + Chrome 开发者工具"></a>工具:NodeJs + Chrome 开发者工具</h2>
                  <p>许久没有水文了，闲来无事特来混混脸熟 <a href="https://github.com/Payne-Wu/PythonScrape/tree/master/JS%E5%88%86%E6%9E%90/%E5%92%AA%E5%92%95" target="_blank" rel="noopener">源码在此，欢迎白嫖，star就更好啦</a></p>
                  <h2 id="enpassword"><a href="#enpassword" class="headerlink" title="enpassword"></a><strong>enpassword</strong></h2>
                  <h3 id="找到登录入口："><a href="#找到登录入口：" class="headerlink" title="找到登录入口："></a>找到登录入口：</h3>
                  <h3 id="查找方式："><a href="#查找方式：" class="headerlink" title="查找方式："></a>查找方式：</h3>
                  <p>点击登录 —&gt; 开启chrome开发者工具 -&gt; 重载框架 —&gt; 抓到<a href="https://passport.migu.cn/login?sourceid=203021&amp;apptype=2&amp;forceAuthn=true&amp;isPassive=false&amp;authType=&amp;display=&amp;nodeId=70027513&amp;relayState=login&amp;weibo=1&amp;callbackURL=http%3A%2F%2Fwww.miguvideo.com%2Fmgs%2Fwebsite%2Fprd%2Findex.html%3FisIframe%3Dweb" target="_blank" rel="noopener">登录包</a> 如下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/1.登录入口查找.png" alt=""></p>
                  <h3 id="加密参数寻找"><a href="#加密参数寻找" class="headerlink" title="加密参数寻找"></a>加密参数寻找</h3>
                  <p>清空之后，使用错误的账号密码登录。一共两个包两张图片。图片开源不看，具体看包，最后在authn包中看到了我们登录所加密过的三个参数，如下 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/2.加密参数寻找.png" alt=""></p>
                  <h3 id="海里捞针-找参数"><a href="#海里捞针-找参数" class="headerlink" title="海里捞针-找参数"></a>海里捞针-找参数</h3>
                  <p>在搜索框(ctrl + shift + F )下搜索enpassword参数，进入source File 发现link 93，name并未加密；那么就是在它的class 属性 J_RsaPsd中。再次找！ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/3.海底捞针-找参数.png" alt=""></p>
                  <h3 id="海里捞针-找参数、埋断点"><a href="#海里捞针-找参数、埋断点" class="headerlink" title="海里捞针-找参数、埋断点"></a>海里捞针-找参数、埋断点</h3>
                  <p>找到三个 J_RsaPsd，每个都上断点，然后在点登录一下 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/3.海底捞针-找参数埋断点.png" alt=""> encrypt：加密函数，b.val加密对象（输入的密码） <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/3.海底捞针-找参数埋断点1.png" alt=""> 将其扣出来！ 为什么扣这里？因为这里为加密处！由明文转为密文。那我们拿到这些就以为着拿到了加密的函数。就可以自己实现加密</p>
                  <blockquote>
                    <p>c = new p.RSAKey; c.setPublic(a.result.modulus, a.result.publicExponent); var d = c.encrypt(b.val());</p>
                  </blockquote>
                  <p>该写如下：（js丫）</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">function</span> get<span class="constructor">Pwd(<span class="params">pwd</span>)</span> &#123;</span><br><span class="line">    c = <span class="keyword">new</span> p.RSAKey;</span><br><span class="line">    c.set<span class="constructor">Public(<span class="params">a</span>.<span class="params">result</span>.<span class="params">modulus</span>, <span class="params">a</span>.<span class="params">result</span>.<span class="params">publicExponent</span>)</span>;</span><br><span class="line">    var d = c.encrypt(b.<span class="keyword">val</span><span class="literal">()</span>);</span><br><span class="line">    return d;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>虽然我们加密的函数已经找到了，but，我们是在自己的环境下并不一定有这个函数（c.encrypt）。所以现在需要去找c.encrypt <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/4.袖里寻针-尝试自己完成加密.png" alt=""> 新问题：p.RSAKey；没有定义；回到chrome进入p.RSAKey-（选中点击进入f db()） <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/4.袖里寻针-尝试自己完成加密-方法p.RSAKey.png" alt=""> 进入f db（）扣出这个方法，然后改写 寻找a.result.modulus, a.result.publicExponent两个参数， 其实是publickey包返回的结果那么至此<strong>enpassword</strong>加密完成 补两个环境参数</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">window</span> = this<span class="comment">;</span></span><br><span class="line"><span class="attr">navigator</span> = &#123;&#125;<span class="comment">;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/5.加密破解完成.png" alt=""></p>
                  <h2 id="fingerPrint、fingerPrintDetail参数破解"><a href="#fingerPrint、fingerPrintDetail参数破解" class="headerlink" title="fingerPrint、fingerPrintDetail参数破解"></a><strong>fingerPrint</strong>、<strong>fingerPrintDetail</strong>参数破解</h2>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/6.换汤不换药-继续.png" alt=""> link480 下断点点击下一步，运行 <strong>运行一步</strong>, 进入<strong>RSAfingerPrint</strong>函数内，把o.page.RSAfingerPrint方法抠出来 在页面中观察a，b参数 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/6.换汤不换药-扣取查看参数.png" alt=""> 观察发现： 其实a，b，就是我们的<strong>a.result.modulus, a.result.publicExponent</strong>，</p>
                  <figure class="highlight javascript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">rsaFingerprint = <span class="function"><span class="keyword">function</span> (<span class="params"></span>) </span>&#123;</span><br><span class="line">    a = <span class="string">"00833c4af965ff7a8409f8b5d5a83d87f2f19d7c1eb40dc59a98d2346cbb145046b2c6facc25b5cc363443f0f7ebd9524b7c1e1917bf7d849212339f6c1d3711b115ecb20f0c89fc2182a985ea28cbb4adf6a321ff7e715ba9b8d7261d1c140485df3b705247a70c28c9068caabbedbf9510dada6d13d99e57642b853a73406817"</span>;</span><br><span class="line">    b = <span class="string">"010001"</span>;</span><br><span class="line">    <span class="keyword">var</span> c = $.fingerprint.details</span><br><span class="line">        , d = $.fingerprint.result</span><br><span class="line">        , e = c.length</span><br><span class="line">        , f = <span class="string">""</span></span><br><span class="line">        , g = <span class="keyword">new</span> m.RSAKey;</span><br><span class="line">    <span class="built_in">console</span>.log(a, b)</span><br><span class="line">    g.setPublic(a, b);</span><br><span class="line">    <span class="keyword">for</span> (<span class="keyword">var</span> h = g.encrypt(d), i = <span class="number">0</span>; e &gt; i; i += <span class="number">117</span>)</span><br><span class="line">        f += g.encrypt(c.substr(i, <span class="number">117</span>));</span><br><span class="line">    <span class="keyword">return</span> &#123;</span><br><span class="line">        details: f,</span><br><span class="line">        result: h</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line">rsaFingerprint()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>继续寻找；这两个</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">c</span> = $.fingerprint.details </span><br><span class="line"><span class="attr">d</span> = $.fingerprint.result</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>浏览器里面测一下，把他从console拿出来 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/6.完成.png" alt=""> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/08/7.完成.png" alt=""></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-08 23:46:13" itemprop="dateCreated datePublished" datetime="2020-08-08T23:46:13+08:00">2020-08-08</time>
                </span>
                <span id="/9549.html" class="post-meta-item leancloud_visitors" data-flag-title="MiGu登录参数分析" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9544.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/9544.html" class="post-title-link" itemprop="url">浅入浅出Hash算法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本文预计阅读需3min</p>
                  <p>你好，我是你老朋友Payne，大家都或许过我之前写的水文-JS解密入门，没看过的童鞋开源回头看看啊。里面主要讲述了Hash MD5的例子，以及加密与解密，相关的。那么今天我们去搞一下MD5的“父亲”， Hash。主要阐述了什么是哈希，哈希运用方向以及hash碰撞及解决方向，请查阅</p>
                  <h2 id="Hash算法"><a href="#Hash算法" class="headerlink" title="Hash算法"></a>Hash算法</h2>
                  <h3 id="哈希"><a href="#哈希" class="headerlink" title="哈希"></a>哈希</h3>
                  <p>（hash）也叫散列。Hash算法，是将一个不定长的输入，通过散列函数变换成一个定长的输出，即散列值。</p>
                  <h3 id="应用"><a href="#应用" class="headerlink" title="应用"></a>应用</h3>
                  <p>Hash主要应用在数据结构以及密码学领域。</p>
                  <h4 id="数据结构："><a href="#数据结构：" class="headerlink" title="数据结构："></a><strong>数据结构：</strong></h4>
                  <blockquote>
                    <p>使用Hash算法的数据结构叫做哈希表，也叫散列表，主要是为了提高查询的效率。它通过把关键码值映射到表中一个位置来访问记录，以加快查找的速度。这个映射函数就是hash function，存放记录的数组叫做哈希表。 在数据结构中应用时，有时需要较高的运算速度而弱化考虑抗碰撞性，可以使用自己构建的哈希函数。</p>
                  </blockquote>
                  <h4 id="密码学："><a href="#密码学：" class="headerlink" title="密码学："></a><strong>密码学：</strong></h4>
                  <blockquote>
                    <p>这种hash散列变换是一种单向运算，具有<strong>不可逆性</strong>即不能根据散列值还原出输入信息，因此严格意义上讲<strong>Hash算法是一种消息摘要算法</strong>，不是一种加密算法。常见的hash算法有：SM3、MD5、SHA-1等 。 在不同的应用场景下，hash函数的选择也会有所侧重。比如在管理数据结构时，主要要考虑运算的快速性，并且要保证hash均匀分布；而应用在密码学中就要优先考虑 抗碰撞性 ，避免出现两段不同明文hash值相同的情况发生。</p>
                  </blockquote>
                  <h3 id="哈希碰撞"><a href="#哈希碰撞" class="headerlink" title="哈希碰撞:"></a>哈希碰撞:</h3>
                  <p>不同的值经过Hash function得到同一值则产生<strong>哈希碰撞</strong>， <strong>防止</strong>哈希碰撞的最有效方法，就是扩大哈希值的取值空间</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">16</span>个二进制位的哈希值，产生碰撞的可能性是 <span class="number">65536</span> 分之一。如果有<span class="number">65537</span>个用户，就一定会产生碰撞。哈希值的长度扩大到<span class="number">32</span>个二进制位，碰撞的可能性就会下降到 <span class="number">4</span>,<span class="number">294</span>,<span class="number">967</span>,<span class="number">296</span> 分之一。</span><br><span class="line"></span><br><span class="line">更长的哈希值意味着更大的存储空间、更多的计算，将影响性能和成本。开发者s必须做出抉择，在安全与成本之间找到平衡。</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="hash碰撞解决办法："><a href="#hash碰撞解决办法：" class="headerlink" title="hash碰撞解决办法："></a>hash碰撞解决办法：</h4>
                  <p><strong>1. 开放地址法</strong></p>
                  <ol>
                    <li>线性探测再散列</li>
                    <li>二次探测再散列</li>
                    <li>
                      <p>伪随机探测再散列</p>
                      <figure class="highlight lsl">
                        <table>
                          <tr>
                            <td class="gutter">
                              <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                            </td>
                            <td class="code">
                              <pre><span class="line">所谓的开放定址法就是一旦发生了冲突，就去寻找下一个空的散列地址，只要散列表足够大，空的散列地址总能找到，并将记录存入 。</span><br><span class="line"></span><br><span class="line">开放寻址法：Hi=(H(<span class="type">key</span>) + di) MOD m,i=<span class="number">1</span>,<span class="number">2</span>,…，k(k&lt;=m<span class="number">-1</span>)，其中H(<span class="type">key</span>)为散列函数，m为散列表长，di为增量序列，可有下列三种取法：</span><br><span class="line"></span><br><span class="line"><span class="number">1</span>)． di=<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,…，m<span class="number">-1</span>，称线性探测再散列；</span><br><span class="line"></span><br><span class="line"><span class="number">2</span>)． di=<span class="number">1</span>^<span class="number">2</span>,(<span class="number">-1</span>)^<span class="number">2</span>,<span class="number">2</span>^<span class="number">2</span>,(<span class="number">-2</span>)^<span class="number">2</span>,(<span class="number">3</span>)^<span class="number">2</span>,…，±(k)^<span class="number">2</span>,(k&lt;=m/<span class="number">2</span>)称二次探测再散列；</span><br><span class="line"></span><br><span class="line"><span class="number">3</span>)． di=伪随机数序列，称伪随机探测再散列。</span><br><span class="line"></span><br><span class="line">用开放定址法解决冲突的做法是：当冲突发生时，使用某种探测技术（线性探测法、二次探测法（解决线性探测的堆积问题）、随机探测法（和二次探测原理一致，不一样的是：二次探测以定值跳跃，而随机探测的散列地址跳跃长度是不定值））在散列表中形成一个探测序列。</span><br><span class="line"></span><br><span class="line">沿此序列逐个单元地查找，直到找到给定的关键字，或者碰到一个开放的地址（即该地址单元为空）为止插入即可。</span><br></pre>
                            </td>
                          </tr>
                        </table>
                      </figure>
                    </li>
                    <li>
                      <p><strong>再哈希法</strong></p>
                    </li>
                  </ol>
                  <blockquote>
                    <p>再哈希法又叫双哈希法，有多个不同的Hash函数，当发生冲突时，使用第二个，第三个，….，等哈希函数去计算地址，直到无冲突。 虽然不易发生聚集，但是增加了计算时间。</p>
                  </blockquote>
                  <ol>
                    <li><strong>链地址法(Java hashmap)</strong></li>
                  </ol>
                  <blockquote>
                    <p>链地址法的基本思想是：每个哈希表节点都有一个next指针，多个哈希表节点可以用next指针构成一个单向链表，将所有关键字为同义词的结点链接在同一个单链表中，</p>
                  </blockquote>
                  <ol>
                    <li><strong>建立公共溢出区</strong></li>
                  </ol>
                  <blockquote>
                    <p>建立公共溢出区的基本思想是：假设哈希函数的值域是 [1,m-1]，则设向量 HashTable[0…m-1] 为基本表， 每个分量存放一个记录，另外设向量 OverTable[0…v] 为溢出表，所有关键字和基本表中关键字为同义词的记录，不管它们由哈希函数得到的哈希地址是什么，一旦发生冲突，都填入溢出表。</p>
                  </blockquote>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-06 02:48:42" itemprop="dateCreated datePublished" datetime="2020-08-06T02:48:42+08:00">2020-08-06</time>
                </span>
                <span id="/9544.html" class="post-meta-item leancloud_visitors" data-flag-title="浅入浅出Hash算法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.6k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9447.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/9447.html" class="post-title-link" itemprop="url">多个站点同时抓取！怎么做到的？</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>你好，我是悦创。 本篇将开启我自己啃代理池的心得，将逐步放送，因为代理池搭建较为复杂，这里我就尽可能把代理池分成几篇来讲，同时也保证，在我其他篇放出来之前，每一篇都是你们的新知识。 学习就像看小说一样，一次一篇就会显得额外的轻松！ 当你把学习当作某个娱乐的事情来做，你会发现不一样的世界！ 我们无法延长生命长度，但是我们延长生命宽度，学习编程就是扩展生命最有力的武器！</p>
                  <h2 id="1-看完之后你会得到什么"><a href="#1-看完之后你会得到什么" class="headerlink" title="1. 看完之后你会得到什么"></a>1. 看完之后你会得到什么</h2>
                  <ul>
                    <li>返回 yield；</li>
                    <li>eval 的使用；</li>
                    <li>多个代理网站同时抓取；</li>
                    <li>使用异步测试代理是否可用；</li>
                    <li>Python 的元类编程简单介绍；</li>
                    <li>正则表达式、PyQuery 提取数据；</li>
                    <li>模块化编程；</li>
                  </ul>
                  <p>废话不多说，马上步入正题！</p>
                  <h2 id="2-你需要的准备"><a href="#2-你需要的准备" class="headerlink" title="2. 你需要的准备"></a>2. 你需要的准备</h2>
                  <p>在学习本篇文章时，希望你已经具备如下技能或者知识点：</p>
                  <ol>
                    <li>Python 环境（推荐 Python 3.7+）；</li>
                    <li>Python 爬虫常用库；</li>
                    <li>Python 基本语法；</li>
                    <li>面向对象编程；</li>
                    <li>yield、eval 的使用；</li>
                    <li>模块化编程；</li>
                  </ol>
                  <h2 id="3-课前预习知识点"><a href="#3-课前预习知识点" class="headerlink" title="3. 课前预习知识点"></a>3. 课前预习知识点</h2>
                  <p>对于代理池的搭建呢，虽然我已经尽可能地照顾到绝大多数地小白，把代理地的搭建呢，进行了拆分，不过对于培训机构或者自学不是特别精进的小伙伴来说还是有些难度的，对于这些呢？我这里也给大家准备了知识点的扫盲。以下内容节选自我个人博客文章，这里为了扫盲就提供必要部分，想看完整的文章可以点击此链接：<a href="https://www.aiyc.top/archives/605.html" target="_blank" rel="noopener">https://www.aiyc.top/archives/605.html</a></p>
                  <h3 id="3-1-Python-的元类编程"><a href="#3-1-Python-的元类编程" class="headerlink" title="3.1 Python 的元类编程"></a>3.1 Python 的元类编程</h3>
                  <p>你好，我是悦创。 好久不见，最近在啃数学、Java、英语没有来更新公众号，那么今天就来更新一下啦！ 又到了每日一啃，啃代码的小悦。今天我遇到了 Python 原类，然而我啥也不懂。只能靠百度和谷歌了，主要还是用谷歌来查啦，百度前几条永远是广告准确度也不行（个人观点）,也顺便参考了几个博客：廖雪峰网站，添加了一点自己的观点和理解。</p>
                  <h4 id="3-1-1-type"><a href="#3-1-1-type" class="headerlink" title="3.1.1 type()"></a>3.1.1 type()</h4>
                  <p>动态语言和静态语言最大的不同，就是函数和类的定义，不是编译时定义的，而是运行时动态创建的。 比方说我们要定义一个 <code>Hello</code> 的 class，就写一个 <code>hello.py</code> 模块（这里我实际创建的是：<code>the_test_code_project.py</code> 模块）：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Hello</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">hello</span><span class="params">(self, name=<span class="string">'world'</span>)</span>:</span></span><br><span class="line">print(<span class="string">'Hello, %s.'</span> % name)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当 Python 解释器载入 <code>hello</code> 模块时，就会依次执行该模块的所有语句，执行结果就是动态创建出一个 <code>Hello</code> 的 class 对象，测试如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="keyword">if</span></span> <span class="variable">__name__</span> == <span class="string">'__main__'</span>:</span><br><span class="line"><span class="variable">h</span> = <span class="function"><span class="title">Hello</span>()</span></span><br><span class="line"><span class="variable">h.hello</span>()</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">Hello</span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">h</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight ceylon">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Hello, world.</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'type'</span>&gt;</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'__main__.Hello'</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中，上面的输出结果：<code>__main__.Hello</code> 等价于 <code>&lt;class &#39;the_test_code_project.Hello&#39;&gt;</code> 运行的方式不同显示的方式也不同，但含义是一样的。 <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200602175929.png" alt=""> <code>type()</code> 函数可以查看一个类型或变量的类型，为了让小白更轻松，我也写了个例子：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">number</span> = <span class="number">12</span></span><br><span class="line"><span class="built_in">string</span> = <span class="string">'Hello AIYC!'</span></span><br><span class="line">float_number = <span class="number">12.1</span></span><br><span class="line">list_data = [<span class="number">1</span>, <span class="string">'2'</span>, <span class="number">3.5</span>, <span class="string">'AIYC'</span>] # 可变</span><br><span class="line">tuples = (<span class="number">1</span>, <span class="string">'2'</span>, <span class="number">3.5</span>, <span class="string">'AIYC'</span>) # 不可变</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(<span class="keyword">number</span>))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(<span class="built_in">string</span>))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(float_number))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(list_data))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(tuples))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;<span class="keyword">class</span> '<span class="symbol">int</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">str</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">float</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">list</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">tuple</span>'&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><code>Hello</code> 是一个 class，它的类型就是 <code>type</code> ，而 <code>h</code> 是一个实例，它的类型就是class <code>Hello</code>。 我们说 class 的定义是运行时动态创建的，而创建 class 的方法就是使用 <code>type()</code> 函数。 <code>type()</code> 函数既可以返回一个对象的类型，又可以创建出新的类型，比如，我们可以通过 <code>type()</code> 函数创建出<code>Hello</code> 类，而无需通过 <code>class Hello(object)...</code> 的定义：</p>
                  <figure class="highlight delphi">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def fn(self, <span class="keyword">name</span>=<span class="string">'world'</span>): # 先定义函数</span><br><span class="line">print(<span class="string">'Hello, %s.'</span> % <span class="keyword">name</span>)</span><br><span class="line"></span><br><span class="line">Hello = <span class="keyword">type</span>(<span class="string">'Hello'</span>, (<span class="keyword">object</span>,), dict(hello=fn)) # 创建 Hello <span class="keyword">class</span></span><br><span class="line"># Hello = <span class="keyword">type</span>(<span class="string">'Class_Name'</span>, (<span class="keyword">object</span>,), dict(hello=fn)) # 创建 Hello <span class="keyword">class</span></span><br><span class="line"># <span class="keyword">type</span>(类名, 父类的元组（针对继承的情况，可以为空），包含属性的字典（名称和值）)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们接下来来调用一下代码，看输出的结果如何：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="keyword">if</span></span> <span class="variable">__name__</span> == <span class="string">'__main__'</span>:</span><br><span class="line"><span class="variable">h</span> = <span class="function"><span class="title">Hello</span>()</span></span><br><span class="line"><span class="variable">h.hello</span>()</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">Hello</span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">h</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里推荐写成：<code>if __name__ == &#39;__main__&#39;:</code> 使代码更加的规范。 运行结果：</p>
                  <figure class="highlight ceylon">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Hello, world.</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'type'</span>&gt;</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'__main__.Hello'</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>要创建一个 class 对象，<code>type()</code> 函数依次传入 3 个参数：</p>
                  <ol>
                    <li>class 的名称；</li>
                    <li>继承的父类集合，注意 Python 支持多重继承，如果只有一个父类，别忘了 tuple 的单元素写法；（这个个 tuple 单元素写法起初本人不太理解，然后一查并认真观察了一下上面的代码就想起来 tuple 单元素写法需要加逗号（，），就是你必须这么写：<code>tuple_1 = (1,)</code> 而不能这么写：<code>tuple_2 = (1)</code> ，<code>tuple_2 = (1)</code> 的写法，Python 会自动认为是一个整数而不是一个元组）</li>
                    <li>class 的方法名称与函数绑定，这里我们把函数 <code>fn</code> 绑定到方法名 <code>hello</code> 上。</li>
                  </ol>
                  <p>通过 <code>type()</code> 函数创建的类和直接写 class 是完全一样的，因为 Python 解释器遇到 class 定义时，仅仅是扫描一下 class 定义的语法，然后调用 <code>type()</code> 函数创建出 class。（直接 Class 创建也是） 正常情况下，我们都用 <code>class Xxx...</code> 来定义类，但是，<code>type()</code> 函数也允许我们动态创建出类来，也就是说，动态语言本身支持运行期动态创建类，这和静态语言有非常大的不同，要在静态语言运行期创建类，必须构造源代码字符串再调用编译器，或者借助一些工具生成字节码实现，本质上都是动态编译，会非常复杂。</p>
                  <h4 id="3-1-2-metaclass"><a href="#3-1-2-metaclass" class="headerlink" title="3.1.2 metaclass"></a>3.1.2 metaclass</h4>
                  <p>除了使用 <code>type()</code> 动态创建类以外，要控制类的 <strong>创建行为</strong> ，还可以使用 metaclass。 metaclass，直译为元类，简单的解释就是：</p>
                  <ul>
                    <li>当我们定义了类以后，就可以根据这个类创建出实例，所以：先定义类，然后创建实例。</li>
                  </ul>
                  <p><img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200602213154.png" alt="image-20200602213149088"></p>
                  <ul>
                    <li>但是如果我们想创建出类呢？</li>
                  </ul>
                  <p>那就必须根据 metaclass 创建出类，所以：先定义 metaclass ，然后创建类。连接起来就是：先定义 metaclass ，就可以创建类，最后创建实例。 所以，<strong>metaclass 允许你创建类或者修改类</strong> 。换句话说，你可以把类看成是 metaclass 创建出来的“实例”。</p>
                  <h6 id="metaclass-是-Python-面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。"><a href="#metaclass-是-Python-面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。" class="headerlink" title="metaclass 是 Python 面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。)"></a>metaclass 是 Python 面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。)</h6>
                  <p>我们先看一个简单的例子，这个 metaclass 可以给我们自定义的 MyList 增加一个 <code>add</code> 方法：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">class</span> <span class="constructor">ListMetaclass(<span class="params">type</span>)</span>:</span><br><span class="line">def <span class="constructor">__new__(<span class="params">cls</span>, <span class="params">name</span>, <span class="params">bases</span>, <span class="params">attrs</span>)</span>:</span><br><span class="line">attrs<span class="literal">['<span class="identifier">add</span>']</span> = lambda self, value: self.append(value) # 加上新的方法</span><br><span class="line">return <span class="keyword">type</span>.<span class="constructor">__new__(<span class="params">cls</span>, <span class="params">name</span>, <span class="params">bases</span>, <span class="params">attrs</span>)</span> # 返回修改后的定义</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <blockquote>
                    <p>定义 <code>ListMetaclass</code> ，按照默认习惯，metaclass 的类名总是以 Metaclass 结尾，以便清楚地表示这是一个metaclass 。</p>
                  </blockquote>
                  <p>有了 ListMetaclass ，我们在定义类的时候还要指示使用 ListMetaclass 来定制类，传入关键字参数 <code>metaclass</code> ：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">class</span> <span class="symbol">MyList</span>(<span class="symbol">list, <span class="symbol">metaclass</span></span>=<span class="symbol">ListMetaclass</span>):</span><br><span class="line"><span class="symbol">pass</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当我们传入关键字参数 <code>metaclass</code> 时，魔术就生效了，它指示 Python 解释器在创建 <code>MyList</code> 时，要通过<code>ListMetaclass.__new__()</code> 来创建，在此，我们可以修改类的定义，比如，加上新的方法，然后，返回修改后的定义。 <code>__new__()</code> 方法接收到的参数依次是：</p>
                  <ol>
                    <li>当前准备创建的类的对象；</li>
                    <li>类的名字；</li>
                    <li>类继承的父类集合；</li>
                    <li>类的方法集合。</li>
                  </ol>
                  <p>测试一下 <code>MyList</code> 是否可以调用 <code>add()</code> 方法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">L = MyList()</span><br><span class="line">L.<span class="builtin-name">add</span>(1)</span><br><span class="line"><span class="builtin-name">print</span>(L)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">[1]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>而普通的 <code>list</code> 没有 <code>add()</code> 方法：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; L2 = <span class="keyword">list</span>()</span><br><span class="line">&gt;&gt;&gt; L2.<span class="built_in">add</span>(<span class="number">1</span>)</span><br><span class="line">Traceback (most recent <span class="keyword">call</span> <span class="keyword">last</span>):</span><br><span class="line">File <span class="string">"&lt;stdin&gt;"</span>, <span class="built_in">line</span> <span class="number">1</span>, in <span class="symbol">&lt;module&gt;</span></span><br><span class="line">AttributeError: <span class="string">'list'</span> object <span class="built_in">has</span> <span class="keyword">no</span> attribute <span class="string">'add'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><strong>这时候，我想你应该和我会有同样的问题，动态修改有什么意义？</strong> <strong>直接在 <code>MyList</code> 定义中写上 <code>add()</code> 方法不是更简单吗？</strong> 正常情况下，确实应该直接写，我觉得通过 metaclass 修改纯属变态。</p>
                  <h3 id="3-2-Python-eval-函数"><a href="#3-2-Python-eval-函数" class="headerlink" title="3.2 Python eval() 函数"></a>3.2 Python eval() 函数</h3>
                  <p>对于 Python 的 eval 呢，大部分人是这么定义的：eval() 函数用来执行一个字符串表达式，并返回表达式的值。 这句话，有可能对于新手来说并不是非常好理解，我们还是用例子来顿悟一下吧。 以下是 eval() 方法的语法：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">eval</span><span class="params">(expression[, globals[, locals]])</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><strong>参数</strong></p>
                  <ul>
                    <li>expression — 表达式。</li>
                    <li>globals — 变量作用域，全局命名空间，如果被提供，则必须是一个字典对象。</li>
                    <li>locals — 变量作用域，局部命名空间，如果被提供，可以是任何映射对象。</li>
                  </ul>
                  <p><strong>返回值</strong> 返回表达式计算结果。 <strong>实际操作</strong></p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">In [<span class="number">3</span>]: x = <span class="number">7</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">4</span>]: eval( <span class="string">'3 * x'</span> )</span><br><span class="line">Out[<span class="number">4</span>]: <span class="number">21</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">5</span>]: eval(<span class="string">'pow(2,2)'</span>)</span><br><span class="line">Out[<span class="number">5</span>]: <span class="number">4</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">6</span>]: eval(<span class="string">'2 + 2'</span>)</span><br><span class="line">Out[<span class="number">6</span>]: <span class="number">4</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">7</span>]: n=<span class="number">81</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">8</span>]: eval(<span class="string">"n + 4"</span>)</span><br><span class="line">Out[<span class="number">8</span>]: <span class="number">85</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再来个函数的操作：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">In</span> [<span class="number">1</span>]: <span class="keyword">str1 </span>= <span class="string">"print('Hello World')"</span></span><br><span class="line"></span><br><span class="line"><span class="symbol">In</span> [<span class="number">2</span>]: eval(<span class="keyword">str1)</span></span><br><span class="line"><span class="keyword">Hello </span>World</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>ok，到此零基础小白关怀文章就完成了，我就不继续赘述啦！ 看到这里，你们的身体还行吗？ <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200616111544" alt="真叫人头大"> 我正在的干货要开始了！ 进入我们的 show time 环节，起飞了、起飞了，做好准备了！</p>
                  <h2 id="4-抓取免费代理"><a href="#4-抓取免费代理" class="headerlink" title="4. 抓取免费代理"></a>4. 抓取免费代理</h2>
                  <p>这里呢，我就不再带带大家手摸手的教学如何抓取代理了，因为当你能看这篇文章时，相信你已经对爬虫是有所入门了，如果还没入门的小伙伴可以关注本公众号，往期文章也有零基础教学和基础的课程资源，可以公众号后台回复：Python爬虫。即可获取资源，如果失效也别担心加小悦好友即可！（资源多多噢！）</p>
                  <blockquote>
                    <p>小白不建议报名培训机构，毕竟现在培训结构收智商税比较多，还是需要多多鉴别噢！</p>
                  </blockquote>
                  <h3 id="4-1-目标的代理网站"><a href="#4-1-目标的代理网站" class="headerlink" title="4.1 目标的代理网站"></a>4.1 目标的代理网站</h3>
                  <ol>
                    <li>快代理：<a href="https://www.kuaidaili.com/" target="_blank" rel="noopener">https://www.kuaidaili.com/</a></li>
                    <li>西刺代理：<a href="https://www.xicidaili.com" target="_blank" rel="noopener">https://www.xicidaili.com</a></li>
                    <li>66代理：<a href="http://www.66ip.cn/" target="_blank" rel="noopener">http://www.66ip.cn/</a></li>
                    <li>无忧代理：<a href="http://www.data5u.com" target="_blank" rel="noopener">http://www.data5u.com</a></li>
                    <li>开心代理-高匿：<a href="http://www.kxdaili.com/dailiip/" target="_blank" rel="noopener">http://www.kxdaili.com/dailiip/</a></li>
                    <li>云代理：<a href="http://www.ip3366.net/free/" target="_blank" rel="noopener">http://www.ip3366.net/free/</a></li>
                  </ol>
                  <p>对于每个网站的代码呢，具体实现也是比较简单的，这里我就不做过多的赘述啦！ 接下来我都直接上代码，不过在上代理的爬虫代码前，我们需要来写个元类，代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ProxyMetaclass</span><span class="params">(type)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">定义 ProxyMetaclass ，按照默认习惯，metaclass 的类名总是以 Metaclass 结尾，以便清楚地表示这是一个metaclass ：</span></span><br><span class="line"><span class="string">元类，在 FreeProxyGetter 类中加入</span></span><br><span class="line"><span class="string">CrawlFunc_list 和 CrawlFuncCount</span></span><br><span class="line"><span class="string">两个参数，分别表示爬虫函数，和爬虫函数的数量。</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__new__</span><span class="params">(cls, name, bases, attrs)</span>:</span></span><br><span class="line">count = <span class="number">0</span></span><br><span class="line">attrs[<span class="string">'CrawlFunc_List'</span>] = [] <span class="comment"># 添加：CrawlFunc_List 列表方法</span></span><br><span class="line"><span class="keyword">for</span> k, v <span class="keyword">in</span> attrs.items():</span><br><span class="line"><span class="keyword">if</span> <span class="string">'crawl_'</span> <span class="keyword">in</span> k:</span><br><span class="line"><span class="comment"># 判断这个函数里面是否携带 crawl_ 也就是利用 value in xxx</span></span><br><span class="line">attrs[<span class="string">'CrawlFunc_List'</span>].append(k)</span><br><span class="line">count += <span class="number">1</span></span><br><span class="line">attrs[<span class="string">'CrawlFuncCount'</span>] = count <span class="comment"># 检测添加的函数的总数量</span></span><br><span class="line"><span class="keyword">return</span> type.__new__(cls, name, bases, attrs) <span class="comment"># 返回所修改的</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>详细的代码含义，已经写在上面了，具体的这里 就不赘述了，如果泥有任何不理解的可以去点击阅读原文在我的博客网站下留言，和后台回复数字“3”，加小编好友，拉你入交流群。</p>
                  <h3 id="4-2-代码编写"><a href="#4-2-代码编写" class="headerlink" title="4.2 代码编写"></a>4.2 代码编写</h3>
                  <h4 id="4-2-1-请求函数编写"><a href="#4-2-1-请求函数编写" class="headerlink" title="4.2.1 请求函数编写"></a>4.2.1 请求函数编写</h4>
                  <p>单独保存成一个文件：<strong>utils.py</strong></p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">project = 'Code', file_name = 'utils.py', author = 'AI悦创'</span></span><br><span class="line"><span class="string">time = '2020/6/1 12:34', product_name = PyCharm, 公众号：AI悦创</span></span><br><span class="line"><span class="string">code is far away from bugs with the god animal protecting</span></span><br><span class="line"><span class="string">I love animals. They taste delicious.</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> aiohttp</span><br><span class="line"><span class="keyword">from</span> requests.exceptions <span class="keyword">import</span> ConnectionError</span><br><span class="line"><span class="keyword">from</span> fake_useragent <span class="keyword">import</span> UserAgent,FakeUserAgentError</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_page</span><span class="params">(url, options=&#123;&#125;)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">构造随机请求头，如果不理解的可以阅读此文章：</span></span><br><span class="line"><span class="string">两行代码设置 Scrapy UserAgent：https://www.aiyc.top/archives/533.html</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">ua = UserAgent()</span><br><span class="line"><span class="keyword">except</span> FakeUserAgentError:</span><br><span class="line"><span class="keyword">pass</span></span><br><span class="line"><span class="comment"># 生成随机的请求头，加 try...except... 使代码更加健壮</span></span><br><span class="line">base_headers = &#123;</span><br><span class="line"><span class="string">'User-Agent'</span>: ua.random,</span><br><span class="line"><span class="string">'Accept-Encoding'</span>: <span class="string">'gzip, deflate, sdch'</span>,</span><br><span class="line"><span class="string">'Accept-Language'</span>: <span class="string">'zh-CN,zh;q=0.8'</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="comment"># 如果使用者有传入请求头，则将此请求头和随机生成的合成在一起</span></span><br><span class="line">headers = dict(base_headers, **options)</span><br><span class="line"><span class="comment"># 当前请求的 url</span></span><br><span class="line">print(<span class="string">'Getting'</span>, url)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">r = requests.get(url, headers=headers)</span><br><span class="line">print(<span class="string">'Getting result'</span>, url, r.status_code)</span><br><span class="line"><span class="keyword">if</span> r.status_code == <span class="number">200</span>:</span><br><span class="line"><span class="keyword">return</span> r.text</span><br><span class="line"><span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"><span class="keyword">except</span> ConnectionError:</span><br><span class="line">print(<span class="string">'Crawling Failed'</span>, url)</span><br><span class="line"><span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Downloader</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">一个异步下载器，可以对代理源异步抓取，但是容易被 BAN。</span></span><br><span class="line"><span class="string">self._htmls: 把请求的 html 放入列表当中</span></span><br><span class="line"><span class="string">升级版的请求类</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, urls)</span>:</span></span><br><span class="line">self.urls = urls</span><br><span class="line">self._htmls = []</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">download_single_page</span><span class="params">(self, url)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">下载单页</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> aiohttp.ClientSession() <span class="keyword">as</span> session:</span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> session.get(url) <span class="keyword">as</span> response:</span><br><span class="line">self._htmls.append(<span class="keyword">await</span> response.text())</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">download</span><span class="params">(self)</span>:</span></span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">tasks = [self.download_single_page(url) <span class="keyword">for</span> url <span class="keyword">in</span> self.urls]</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"></span><br><span class="line"><span class="meta">@property</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">htmls</span><span class="params">(self)</span>:</span></span><br><span class="line">self.download()</span><br><span class="line"><span class="keyword">return</span> self._htmls</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>上面请求函数编写完成之后，我们就需要在抓取代理的代码文件中，进行导包，代码如下：</p>
                  <figure class="highlight capnproto">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># files:Spider.py</span></span><br><span class="line"><span class="keyword">from</span> utils <span class="keyword">import</span> get_page</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="4-2-2-代理抓取代码"><a href="#4-2-2-代理抓取代码" class="headerlink" title="4.2.2 代理抓取代码"></a>4.2.2 代理抓取代码</h4>
                  <p>导入所需要的库：</p>
                  <figure class="highlight elm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="title">from</span> utils <span class="keyword">import</span> get_page</span><br><span class="line"><span class="title">from</span> pyquery <span class="keyword">import</span> PyQuery <span class="keyword">as</span> pq</span><br><span class="line"><span class="keyword">import</span> re</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们需要创建一个类 FreeProxyGetter 并使用元类创建。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">class</span> <span class="symbol">FreeProxyGetter</span>(<span class="symbol">object, <span class="symbol">metaclass</span></span>=<span class="symbol">ProxyMetaclass</span>):</span><br><span class="line"><span class="symbol">pass</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h5 id="1-快代理"><a href="#1-快代理" class="headerlink" title="1. 快代理"></a>1. 快代理</h5>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_kuaidaili</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">快代理</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">4</span>):</span><br><span class="line"><span class="comment"># 国内高匿代理</span></span><br><span class="line">start_url = <span class="string">'https://www.kuaidaili.com/free/inha/&#123;&#125;/'</span>.format(page)</span><br><span class="line">html = get_page(start_url)</span><br><span class="line"><span class="comment"># print(html)</span></span><br><span class="line">pattern = re.compile(</span><br><span class="line"><span class="string">'&lt;td data-title="IP"&gt;(.*)&lt;/td&gt;s*&lt;td data-title="PORT"&gt;(w+)&lt;/td&gt;'</span></span><br><span class="line">)</span><br><span class="line"><span class="comment"># s * 匹配空格，起到换行作用</span></span><br><span class="line"><span class="comment"># ip_addres = re.findall(pattern, html) # 写法一</span></span><br><span class="line">ip_addres = pattern.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> ip_addres:</span><br><span class="line"><span class="comment"># print(adress, port)</span></span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span>.strip()</span><br><span class="line"><span class="keyword">yield</span> result</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h5 id="2-西刺代理"><a href="#2-西刺代理" class="headerlink" title="2. 西刺代理"></a>2. 西刺代理</h5>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_xicidaili</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">西刺代理</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">4</span>):</span><br><span class="line">start_url = <span class="string">'https://www.xicidaili.com/wt/&#123;&#125;'</span>.format(page)</span><br><span class="line">html = get_page(start_url)</span><br><span class="line"><span class="comment"># print(html)</span></span><br><span class="line">ip_adress = re.compile(</span><br><span class="line"><span class="string">'&lt;td class="country"&gt;&lt;img src="//fs.xicidaili.com/images/flag/cn.png" alt="Cn" /&gt;&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;'</span></span><br><span class="line">)</span><br><span class="line"><span class="comment"># s* 匹配空格，起到换行作用</span></span><br><span class="line">re_ip_adress = ip_adress.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_adress:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span>.strip()</span><br><span class="line"><span class="keyword">yield</span> result</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h5 id="3-66代理"><a href="#3-66代理" class="headerlink" title="3. 66代理"></a>3. 66代理</h5>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def crawl_daili66(self, <span class="attribute">page_count</span>=4):</span><br><span class="line"><span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">66代理</span></span><br><span class="line"><span class="string">:param page_count:</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"</span><span class="string">""</span></span><br><span class="line">start_url = <span class="string">'http://www.66ip.cn/&#123;&#125;.html'</span></span><br><span class="line">urls = [start_url.format(page) <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(1, page_count + 1)]</span><br><span class="line"><span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Crawling'</span>, url)</span><br><span class="line">html = get_page(url)</span><br><span class="line"><span class="keyword">if</span> html:</span><br><span class="line">doc = pq(html)</span><br><span class="line">trs = doc(<span class="string">'.containerbox table tr:gt(0)'</span>).items()</span><br><span class="line"><span class="keyword">for</span> tr <span class="keyword">in</span> trs:</span><br><span class="line">ip = tr.<span class="builtin-name">find</span>(<span class="string">'td:nth-child(1)'</span>).text()</span><br><span class="line">port = tr.<span class="builtin-name">find</span>(<span class="string">'td:nth-child(2)'</span>).text()</span><br><span class="line">yield <span class="string">':'</span>.join([ip, port])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h5 id="4-无忧代理"><a href="#4-无忧代理" class="headerlink" title="4. 无忧代理"></a>4. 无忧代理</h5>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_data5u</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">无忧代理</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">start_url = <span class="string">'http://www.data5u.com'</span></span><br><span class="line">html = get_page(start_url)</span><br><span class="line"><span class="comment"># print(html)</span></span><br><span class="line">ip_adress = re.compile(</span><br><span class="line"><span class="string">'&lt;ul class="l2"&gt;s*&lt;span&gt;&lt;li&gt;(.*?)&lt;/li&gt;&lt;/span&gt;s*&lt;span style="width: 100px;"&gt;&lt;li class=".*"&gt;(.*?)&lt;/li&gt;&lt;/span&gt;'</span></span><br><span class="line">)</span><br><span class="line"><span class="comment"># s * 匹配空格，起到换行作用</span></span><br><span class="line">re_ip_adress = ip_adress.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_adress:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span></span><br><span class="line"><span class="keyword">yield</span> result.strip()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h5 id="5-开心代理-高匿"><a href="#5-开心代理-高匿" class="headerlink" title="5. 开心代理-高匿"></a>5. 开心代理-高匿</h5>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_kxdaili</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">开心代理-高匿</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">4</span>):</span><br><span class="line">start_url = <span class="string">'http://www.kxdaili.com/dailiip/1/&#123;&#125;.html'</span>.format(i)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">html = requests.get(start_url)</span><br><span class="line"><span class="keyword">if</span> html.status_code == <span class="number">200</span>:</span><br><span class="line">html.encoding = <span class="string">'utf-8'</span></span><br><span class="line"><span class="comment"># print(html.text)</span></span><br><span class="line">ip_adress = re.compile(<span class="string">'&lt;tr.*?&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;'</span>)</span><br><span class="line"><span class="comment"># s* 匹配空格，起到换行作用</span></span><br><span class="line">re_ip_adress = ip_adress.findall(str(html.text))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_adress:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span></span><br><span class="line"><span class="keyword">yield</span> result.strip()</span><br><span class="line"><span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"><span class="keyword">except</span>:</span><br><span class="line"><span class="keyword">pass</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h5 id="6-云代理"><a href="#6-云代理" class="headerlink" title="6. 云代理"></a>6. 云代理</h5>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">IP3366Crawler</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">云代理</span></span><br><span class="line"><span class="string">parse html file to get proxies</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">start_url = <span class="string">'http://www.ip3366.net/free/?stype=1&amp;page=&#123;page&#125;'</span></span><br><span class="line">urls = [start_url.format(page=i) <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">8</span>)]</span><br><span class="line"><span class="comment"># s * 匹配空格，起到换行作用</span></span><br><span class="line">ip_address = re.compile(<span class="string">'&lt;tr&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;'</span>)</span><br><span class="line"><span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">html = get_page(url)</span><br><span class="line">re_ip_address = ip_address.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_address:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span></span><br><span class="line"><span class="keyword">yield</span> result.strip()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>至此，代理网站的代码已经全部编写完成，接下来我们需要了解的是然后调用此类，直接调用吗？ 显然不是，我们还需要编写一个运行调用的函数，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def <span class="builtin-name">run</span>(self):</span><br><span class="line"><span class="comment"># print(self.__CrawlFunc__)</span></span><br><span class="line">proxies = []</span><br><span class="line">callback = self.CrawlFunc_List</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> callback:</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Callback'</span>, i)</span><br><span class="line"><span class="keyword">for</span><span class="built_in"> proxy </span><span class="keyword">in</span> eval(<span class="string">"self.&#123;&#125;()"</span>.format(i)):</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Getting'</span>, proxy, <span class="string">'from'</span>, i)</span><br><span class="line">proxies.append(proxy)</span><br><span class="line">return proxies</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上代理的代码中，我并没有抓取每个网站全部页数，如果有需要可以自行调节。 这里我们可以直接写一个调用代码，来测试代码是否正常，写在：<strong>Spider.py</strong> 也就是代理的代码中，代码如下：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">if __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">Tester = FreeProxyGetter()</span><br><span class="line">Tester.<span class="keyword">run</span><span class="bash">()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下： 结果较多，省略大部分结果。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Getting <span class="number">14.115</span><span class="number">.70</span><span class="number">.177</span>:<span class="number">4216</span> <span class="keyword">from</span> crawl_xicidaili</span><br><span class="line">Getting <span class="number">116.22</span><span class="number">.48</span><span class="number">.220</span>:<span class="number">4216</span> <span class="keyword">from</span> crawl_xicidaili</span><br><span class="line">Getting <span class="number">223.241</span><span class="number">.3</span><span class="number">.120</span>:<span class="number">4216</span> <span class="keyword">from</span> crawl_xicidaili</span><br><span class="line">......</span><br><span class="line">Getting <span class="number">60.188</span><span class="number">.1</span><span class="number">.27</span>:<span class="number">8232</span> <span class="keyword">from</span> crawl_data5u</span><br><span class="line">Callback crawl_kxdaili</span><br><span class="line">Getting <span class="number">117.71</span><span class="number">.165</span><span class="number">.208</span>:<span class="number">3000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">223.99</span><span class="number">.197</span><span class="number">.253</span>:<span class="number">63000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">60.255</span><span class="number">.186</span><span class="number">.169</span>:<span class="number">8888</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">180.168</span><span class="number">.13</span><span class="number">.26</span>:<span class="number">8000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">125.124</span><span class="number">.51</span><span class="number">.226</span>:<span class="number">808</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">47.99</span><span class="number">.145</span><span class="number">.67</span>:<span class="number">808</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">222.186</span><span class="number">.55</span><span class="number">.41</span>:<span class="number">8080</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">36.6</span><span class="number">.224</span><span class="number">.30</span>:<span class="number">3000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">110.43</span><span class="number">.42</span><span class="number">.200</span>:<span class="number">8081</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">39.137</span><span class="number">.69</span><span class="number">.8</span>:<span class="number">80</span> <span class="keyword">from</span> crawl_kxdaili</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>ok，至此代理成功抓取，接下来就是我们的异步检测了。</p>
                  <h4 id="4-2-3-异步检测"><a href="#4-2-3-异步检测" class="headerlink" title="4.2.3 异步检测"></a>4.2.3 异步检测</h4>
                  <p>这里，我们需要编写一个 <strong>error.py</strong> 来自定义包：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ResourceDepletionError</span>(<span class="title">Exception</span>):</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">Exception.__init_<span class="number">_</span>(<span class="keyword">self</span>)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__str__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line"><span class="keyword">return</span> repr(<span class="string">'The proxy source is exhausted'</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">PoolEmptyError</span>(<span class="title">Exception</span>):</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">Exception.__init_<span class="number">_</span>(<span class="keyword">self</span>)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__str__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line"><span class="keyword">return</span> repr(<span class="string">'The proxy pool is empty'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>异步检测代码，这里我就不做任何数据存储，有需求的可以自行添加，下一篇将添加一个存入 redis 数据库的，敬请关注！</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># files:schedule.py</span></span><br><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> multiprocessing.context <span class="keyword">import</span> Process</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> aiohttp</span><br><span class="line"><span class="keyword">from</span> db <span class="keyword">import</span> RedisClient</span><br><span class="line"><span class="keyword">from</span> Spider <span class="keyword">import</span> FreeProxyGetter</span><br><span class="line"><span class="keyword">from</span> error <span class="keyword">import</span> ResourceDepletionError</span><br><span class="line"><span class="keyword">from</span> aiohttp <span class="keyword">import</span> ServerDisconnectedError, ClientResponseError, ClientConnectorError</span><br><span class="line"><span class="keyword">from</span> socks <span class="keyword">import</span> ProxyConnectionError</span><br><span class="line"></span><br><span class="line">get_proxy_timeout = <span class="number">9</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ValidityTester</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">代理的有效性测试</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">self.raw_proxies = <span class="literal">None</span></span><br><span class="line">self.usable_proxies = [] <span class="comment"># 可用代理</span></span><br><span class="line">self.test_api = <span class="string">'http://www.baidu.com'</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">set_raw_proxies</span><span class="params">(self, proxies)</span>:</span></span><br><span class="line">self.raw_proxies = proxies <span class="comment"># 临时存储一些代理</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">test_single_proxy</span><span class="params">(self, proxy)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">python3.5 之后出现的新特性</span></span><br><span class="line"><span class="string">text one proxy, if valid, put them to usable_proxies.</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> aiohttp.ClientSession() <span class="keyword">as</span> session:</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line"><span class="keyword">if</span> isinstance(proxy, bytes):</span><br><span class="line">proxy = proxy.decode(<span class="string">'utf-8'</span>)</span><br><span class="line">real_proxy = <span class="string">'http://'</span> + proxy</span><br><span class="line">print(<span class="string">'Testing'</span>, proxy)</span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> session.get(self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) <span class="keyword">as</span> response:</span><br><span class="line"><span class="keyword">if</span> response.status == <span class="number">200</span>:</span><br><span class="line">print(<span class="string">'Valid proxy'</span>, proxy)</span><br><span class="line"><span class="keyword">except</span> (ProxyConnectionError, TimeoutError, ValueError):</span><br><span class="line">print(<span class="string">'Invalid proxy'</span>, proxy)</span><br><span class="line"><span class="keyword">except</span> (ServerDisconnectedError, ClientResponseError, ClientConnectorError) <span class="keyword">as</span> s:</span><br><span class="line">print(s)</span><br><span class="line"><span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">test</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">aio test all proxies.</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">print(<span class="string">'ValidityTester is working'</span>)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">tasks = [self.test_single_proxy(proxy) <span class="keyword">for</span> proxy <span class="keyword">in</span> self.raw_proxies]</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"><span class="keyword">except</span> ValueError:</span><br><span class="line">print(<span class="string">'Async Error'</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Schedule</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">valid_proxy</span><span class="params">()</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">Get half of proxies which in Spider</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">tester = ValidityTester()</span><br><span class="line">free_proxy_getter = FreeProxyGetter()</span><br><span class="line"><span class="comment"># 获取 Spider 数据库数据</span></span><br><span class="line"><span class="comment"># 调用测试类</span></span><br><span class="line">tester.set_raw_proxies(free_proxy_getter.run())</span><br><span class="line"><span class="comment"># 开始测试</span></span><br><span class="line">tester.test()</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">run</span><span class="params">(self)</span>:</span></span><br><span class="line">print(<span class="string">'Ip processing running'</span>)</span><br><span class="line">valid_process = Process(target=Schedule.valid_proxy) <span class="comment"># 获取代理并筛选</span></span><br><span class="line">valid_process.start()</span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">schedule = Schedule().run()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>对于上面的部分代理代码，我为了输出结果更加直观，把 <strong>print()</strong> 输出的，包括 <strong>utils.py</strong> 的 <strong>print()</strong> 全部注释了， 运行结果（省略部分输出）</p>
                  <figure class="highlight subunit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Testing </span>182.46.252.84:9999</span><br><span class="line"><span class="keyword">Testing </span>183.63.188.250:8808</span><br><span class="line"><span class="keyword">Testing </span>39.137.69.10:8080</span><br><span class="line"><span class="keyword">Testing </span>119.190.149.226:8060</span><br><span class="line"><span class="keyword">Testing </span>219.159.38.201:56210</span><br><span class="line"><span class="keyword">Testing </span>58.246.143.32:8118</span><br><span class="line"><span class="keyword">Testing </span>114.99.117.13:3000</span><br><span class="line"><span class="keyword">Testing </span>114.229.229.248:8118</span><br><span class="line"><span class="keyword">Testing </span>119.57.108.53:53281</span><br><span class="line"><span class="keyword">Testing </span>223.247.94.164:4216</span><br><span class="line"><span class="keyword">Testing </span>219.159.38.208:56210</span><br><span class="line"><span class="keyword">Testing </span>59.44.78.30:42335</span><br><span class="line"><span class="keyword">Testing </span>112.74.87.172:3128</span><br><span class="line"><span class="keyword">Testing </span>182.92.235.109:3128</span><br><span class="line"><span class="keyword">Testing </span>58.62.115.3:4216</span><br><span class="line"><span class="keyword">Testing </span>183.162.171.37:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.5.46:4216</span><br><span class="line"><span class="keyword">Testing </span>223.243.4.231:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.5.228:4216</span><br><span class="line"><span class="keyword">Testing </span>183.162.171.56:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.4.146:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.4.200:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.4.182:4216</span><br><span class="line"><span class="keyword">Testing </span>118.187.50.114:8080</span><br><span class="line"><span class="keyword">Testing </span>223.214.176.170:3000</span><br><span class="line"><span class="keyword">Testing </span>125.126.123.208:60004</span><br><span class="line"><span class="keyword">Testing </span>163.125.248.39:8088</span><br><span class="line"><span class="keyword">Testing </span>119.57.108.89:53281</span><br><span class="line"><span class="keyword">Testing </span>222.249.238.138:8080</span><br><span class="line"><span class="keyword">Testing </span>125.126.97.77:60004</span><br><span class="line"><span class="keyword">Testing </span>222.182.54.210:8118</span><br><span class="line">Valid proxy 58.220.95.90:9401</span><br><span class="line">Valid proxy 60.191.45.92:8080</span><br><span class="line">Valid proxy 222.186.55.41:8080</span><br><span class="line">Valid proxy 60.205.132.71:80</span><br><span class="line">Valid proxy 124.90.49.146:8888</span><br><span class="line"></span><br><span class="line">Valid proxy 218.75.109.86:3128</span><br><span class="line">Valid proxy 27.188.64.70:8060</span><br><span class="line"></span><br><span class="line">Valid proxy 27.188.62.3:8060</span><br><span class="line">Valid proxy 221.180.170.104:8080</span><br><span class="line">Valid proxy 39.137.69.9:8080</span><br><span class="line"></span><br><span class="line">Valid proxy 39.137.69.6:80</span><br><span class="line">Valid proxy 39.137.69.10:8080</span><br><span class="line">Valid proxy 114.229.6.215:8118</span><br><span class="line">Valid proxy 221.180.170.104:8080</span><br><span class="line">Valid proxy 218.89.14.142:8060</span><br><span class="line">Valid proxy 116.196.85.150:3128</span><br><span class="line"></span><br><span class="line">Valid proxy 122.224.65.197:3128</span><br><span class="line">Valid proxy 112.253.11.103:8000</span><br><span class="line">Valid proxy 112.253.11.103:8000</span><br><span class="line">Valid proxy 112.253.11.113:8000</span><br><span class="line">Valid proxy 124.90.52.200:8888</span><br><span class="line"></span><br><span class="line">Cannot connect to host 113.103.226.99:4216 ssl:default [Connect call failed ('113.103.226.99', 4216)]</span><br><span class="line">Cannot connect to host 36.249.109.59:8221 ssl:default [Connect call failed ('36.249.109.59', 8221)]</span><br><span class="line">Cannot connect to host 125.126.121.66:60004 ssl:default [Connect call failed ('125.126.121.66', 60004)]</span><br><span class="line">Cannot connect to host 27.42.168.46:48919 ssl:default [Connect call failed ('27.42.168.46', 48919)]</span><br><span class="line">Cannot connect to host 163.125.113.220:8118 ssl:default [Connect call failed ('163.125.113.220', 8118)]</span><br><span class="line">Cannot connect to host 114.55.63.34:808 ssl:default [Connect call failed ('114.55.63.34', 808)]</span><br><span class="line">Cannot connect to host 119.133.207.10:4216 ssl:default [Connect call failed ('119.133.207.10', 4216)]</span><br><span class="line">Cannot connect to host 58.62.115.3:4216 ssl:default [Connect call failed ('58.62.115.3', 4216)]</span><br><span class="line">Cannot connect to host 117.71.164.232:3000 ssl:default [Connect call failed ('117.71.164.232', 3000)]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>到此，多站抓取异步检测，就已经完全实现，也是实现了模块化编程。我也 顺便把项目结构分享一下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">C</span><span class="selector-pseudo">:.</span></span><br><span class="line">│ <span class="selector-tag">error</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">info</span><span class="selector-class">.txt</span></span><br><span class="line">│ <span class="selector-tag">schedule</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">Spider</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">the_test_code_crawler</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">utils</span><span class="selector-class">.py</span></span><br><span class="line">│</span><br><span class="line">└─__<span class="selector-tag">pycache__</span></span><br><span class="line"><span class="selector-tag">db</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br><span class="line"><span class="selector-tag">error</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br><span class="line"><span class="selector-tag">Spider</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br><span class="line"><span class="selector-tag">utils</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>本篇我想你主要学习到的目标也就是开头所说的，这里就不赘述了，下一篇我将带大家把抓取到可用的代理进行存储到 redis 数据库中，尽请关注！ 源代码文件获取：公众号后台回放：ProxyPool-1 也欢迎加入我的交流群，一起交流！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/黄家宝|AI悦创" class="author" itemprop="url" rel="index">黄家宝|AI悦创</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-02 17:24:34" itemprop="dateCreated datePublished" datetime="2020-08-02T17:24:34+08:00">2020-08-02</time>
                </span>
                <span id="/9447.html" class="post-meta-item leancloud_visitors" data-flag-title="多个站点同时抓取！怎么做到的？" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>17k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>15 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9533.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9533.html" class="post-title-link" itemprop="url">Pyppeteer 如何打包 Docker</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>之前开发了一个工具包 GerapyPyppeteer，GitHub 地址为 <a href="https://github.com/Gerapy/GerapyPyppeteer，这个包实现了" target="_blank" rel="noopener">https://github.com/Gerapy/GerapyPyppeteer，这个包实现了</a> Scrapy 和 Pyppeteer 的对接，利用它我们就可以方便地实现 Scrapy 使用 Pyppeteer 爬取动态渲染的页面了。 另外，很多朋友在运行爬虫的时候可能会使用到 Docker，想把 Scrapy 和 Pyppeteer 打包成 Docker 运行，但是这个打包和测试过程中大家可能会遇到一些问题，在这里对 Pyppeteer 打包 Docker 的坑简单做一下总结。</p>
                  <h2 id="概述"><a href="#概述" class="headerlink" title="概述"></a>概述</h2>
                  <p>Pyppeteer 打包 Docker 主要是有这么几个坑点：</p>
                  <ul>
                    <li>依赖没有安装，导致无法正确安装和启动 Pyppeteer。</li>
                    <li>没有关闭沙盒模式，导致可能出现 Browser closed unexpectedly 错误</li>
                    <li>没有提前安装好 Pyppeteer，导致每次启动时都要重新安装</li>
                  </ul>
                  <p>下面我们分别对三个问题做下简单的 Troubleshooting。</p>
                  <h2 id="安装依赖"><a href="#安装依赖" class="headerlink" title="安装依赖"></a>安装依赖</h2>
                  <p>首先说第一个，安装依赖。 因为 Docker 大部分都是基于 Linux 系统的，比如我常用的基础镜像就是 <code>python:3.7</code>，剩余 Debian 系列，当然还有很多其他的版本，具体可以查看 <a href="https://hub.docker.com/_/python" target="_blank" rel="noopener">https://hub.docker.com/_/python</a> 了解下。 但是对于 Pyppeteer 来说，<code>python:3.7</code> 内置的依赖库并不够，我们还需要额外进行安装，安装完毕之后还需要清空下 apt list，一句 Dockerfile 命令如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">RUN apt-<span class="keyword">get</span> update &amp;&amp; </span><br><span class="line">  apt-<span class="keyword">get</span> -y install libnss3 xvfb gconf-service libasound2 libatk1<span class="number">.0</span><span class="number">-0</span> libc6 libcairo2 libcups2 </span><br><span class="line">  libdbus<span class="number">-1</span><span class="number">-3</span> libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf<span class="number">-2</span><span class="number">-4</span> libgdk-pixbuf2<span class="number">.0</span><span class="number">-0</span> libglib2<span class="number">.0</span><span class="number">-0</span> </span><br><span class="line">  libgtk<span class="number">-3</span><span class="number">-0</span> libnspr4 libpango<span class="number">-1.0</span><span class="number">-0</span> libpangocairo<span class="number">-1.0</span><span class="number">-0</span> libstdc++<span class="number">6</span> libx11<span class="number">-6</span> libx11-xcb1 libxcb1 </span><br><span class="line">  libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 </span><br><span class="line">  libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget &amp;&amp; </span><br><span class="line">  rm -rf /var/lib/apt/lists<span class="comment">/*</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="关闭沙盒模式"><a href="#关闭沙盒模式" class="headerlink" title="关闭沙盒模式"></a>关闭沙盒模式</h2>
                  <p>在 Docker 中如果直接启动 Pyppeteer，我们还需要关闭沙盒模式，否则可能会遇到如下错误：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pyppeteer.errors.BrowserError: Browser closed unexpectedly:</span><br><span class="line">[<span class="number">0924</span>/<span class="number">153706.301300</span>:ERROR:zygote_host_impl_linux.<span class="keyword">cc</span>(<span class="number">89</span>)] Running <span class="keyword">as</span> root without --<span class="keyword">no</span>-<span class="keyword">sandbox</span> <span class="keyword">is</span> not supported. See http<span class="variable">s:</span>//crbug.<span class="keyword">com</span>/<span class="number">638180</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里提示我们要关闭沙盒模式，这里只需要在启动 Pyppeteer 的时候，给 launch 方法的 args 参数多加一个 <code>\--no-sandbox</code> 即可，写法如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">browser</span> = await pyppeteer.launch(options=&#123;<span class="string">'args'</span>: [<span class="string">'--no-sandbox'</span>]&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就不会再遇到上面的错误了。</p>
                  <h2 id="提前安装"><a href="#提前安装" class="headerlink" title="提前安装"></a>提前安装</h2>
                  <p>另外建议在打包 Docker 的时候就提前把 Pyppeteer 提前安装好，可以单独使用一句 RUN 命令安装即可。</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">RUN pip <span class="keyword">install</span> -U pip &amp;&amp; pip <span class="keyword">install</span> pyppeteer &amp;&amp; pyppeteer-<span class="keyword">install</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里是提前安装了一下 Pyppteer 这个 Python 库，然后利用 Python 库提供的 pyppeteer-install 命令提前下载了 Chromium 浏览器。 这样后面启动的时候就可以直接唤起 Chromium 浏览器进行爬取了。</p>
                  <h2 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h2>
                  <p>最后看下完整 Dockerfile，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">FROM python:<span class="number">3.7</span></span><br><span class="line"></span><br><span class="line">RUN apt-<span class="keyword">get</span> update &amp;&amp; </span><br><span class="line">  apt-<span class="keyword">get</span> -y install libnss3 xvfb gconf-service libasound2 libatk1<span class="number">.0</span><span class="number">-0</span> libc6 libcairo2 libcups2 </span><br><span class="line">  libdbus<span class="number">-1</span><span class="number">-3</span> libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf<span class="number">-2</span><span class="number">-4</span> libgdk-pixbuf2<span class="number">.0</span><span class="number">-0</span> libglib2<span class="number">.0</span><span class="number">-0</span> </span><br><span class="line">  libgtk<span class="number">-3</span><span class="number">-0</span> libnspr4 libpango<span class="number">-1.0</span><span class="number">-0</span> libpangocairo<span class="number">-1.0</span><span class="number">-0</span> libstdc++<span class="number">6</span> libx11<span class="number">-6</span> libx11-xcb1 libxcb1 </span><br><span class="line">  libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 </span><br><span class="line">  libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget &amp;&amp; </span><br><span class="line">  rm -rf /var/lib/apt/lists<span class="comment">/*</span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment">RUN pip install -U pip &amp;&amp; pip install pyppeteer &amp;&amp; pyppeteer-install</span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment">WORKDIR /code</span></span><br><span class="line"><span class="comment">COPY requirements.txt .</span></span><br><span class="line"><span class="comment">RUN pip install -r requirements.txt</span></span><br><span class="line"><span class="comment">COPY . .</span></span><br><span class="line"><span class="comment">CMD python3 run.py</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先就是安装了必须的依赖库，然后安装了 Pyppeteer 并下载了 Chromium 浏览器，最后拷贝项目运行即可。 当然最后的一句 CMD 大家可以随意指定入口。 最后大家可以体验一个实例来感受下 Scrapy 和 Pyppeteer 对接后在 Docker 中的运行效果：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">docker <span class="keyword">run</span><span class="bash"> germey/gerapy-pyppeteer-example</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果大家对 Scrapy 和 Pyppeteer 感兴趣也可以看下我写的这个库 GerapyPyppeteer，GitHub 地址为 <a href="https://github.com/Gerapy/GerapyPyppeteer，感谢支持" target="_blank" rel="noopener">https://github.com/Gerapy/GerapyPyppeteer，感谢支持</a>！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-02 17:20:35" itemprop="dateCreated datePublished" datetime="2020-08-02T17:20:35+08:00">2020-08-02</time>
                </span>
                <span id="/9533.html" class="post-meta-item leancloud_visitors" data-flag-title="Pyppeteer 如何打包 Docker" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9531.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9531.html" class="post-title-link" itemprop="url">如何区分一个页面是列表页还是详情页</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>解析页面是做爬虫的过程中的重要环节，而且如果站点多了，解析也会变得非常复杂，所以智能化解析就可能是一个不错的解决方案。如果我们能够容忍一定的错误率，那么我们可以利用智能化解析算法帮我们提取一些内容，简单高效。 那有没有办法做到一个网站的全自动化解析呢？ 比如来了一个博客网站，我能首先识别出来这是一个列表页还是文章（详情）页，然后提取列表页的每篇文章的链接，然后跳转到每篇文章（详情）页再提取文章相关信息。 那么这里面可能就有四个关键部分：</p>
                  <ul>
                    <li>判断当前所在的页面是列表页还是文章（详情）页</li>
                    <li>识别出列表页下一页的链接</li>
                    <li>识别出列表页所有列表链接</li>
                    <li>识别出文章（详情）页的文章内容和其他信息</li>
                  </ul>
                  <p>如果我们能把这四步都用算法实现出来，那么我们只需要一个网站的主站链接就能轻松地把内容规整地爬取下来了。 那么这篇文章我们就来简单说下第一步，如何判断当前所在的页面的列表页还是文章（详情）页。</p>
                  <blockquote>
                    <p>注：后文中文章页统一称之为详情页。</p>
                  </blockquote>
                  <h2 id="示例"><a href="#示例" class="headerlink" title="示例"></a>示例</h2>
                  <p>列表页和详情页不知道大家有没有基本的概念了，列表页就是导航页，里面带有好多文章或新闻或详情链接，我们选一个链接点进去就是详情页。 比如说这里拿新浪体育来说，首页如图所示： <img src="https://qiniu.cuiqingcai.com/f8u1z.png" alt="image-20200720193407237"> 看到这里面有很多链接，就是一些页面导航集合，这个页面就是列表页。 然后我们随便点开一篇新闻报道，如图所示： <img src="https://qiniu.cuiqingcai.com/e0w4d.png" alt="image-20200720193504042"> 这里就是一篇新闻报道，带有醒目的标题、发布时间、正文等内容，这就是详情页。 现在我们要做的就是用一个算法来凭借 HTML 代码区分出来哪个是列表页，哪个是详情页。 最后的输入输出如下：</p>
                  <ul>
                    <li>输入：一个页面的 HTML 代码</li>
                    <li>输出：这个页面是列表页还是详情页，并输出二者的判定概率。</li>
                  </ul>
                  <h2 id="模型选用"><a href="#模型选用" class="headerlink" title="模型选用"></a>模型选用</h2>
                  <p>首先我们确认下这个问题是个什么问题。 很明显，结果要么是列表页，要么是详情页，就是个二分类问题。 那二分类问题怎么解决呢？实现一个基本的分类模型就好了。大范围就是传统机器学习和现在比较流行的深度学习。总体上来说，深度学习的精度和处理能力会强一点，但是想想我们现在的应用场景，后者要追求精度的话可能需要更多的标注数据，而前者也有比较不错的易用的模型了，比如 SVM。 所以，我们不妨先选用 SVM 模型来实现一个基本的二分类模型来试试看，效果如果已经很好了或者提升空间不大了，那就直接用就好了，如果效果比较差，那我们再选用其他模型来优化。 好，那就定下来了，我们用 SVM 模型来实现一下试试。</p>
                  <h2 id="数据标注"><a href="#数据标注" class="headerlink" title="数据标注"></a>数据标注</h2>
                  <p>既然要做分类模型，那么最重要的当然就是数据标注了，我们分两组就好了，一组是列表页，一组是详情页，我们先用手工配合爬虫找一些列表页和详情页的 HTML 代码，然后将其保存下来。 结果类似如下： <img src="https://qiniu.cuiqingcai.com/ejxws.png" alt="image-20200720195132784"> 每个文件夹几百个就行了，数量不用太多，五花八门的页面混起来更好。</p>
                  <h2 id="特征提取"><a href="#特征提取" class="headerlink" title="特征提取"></a>特征提取</h2>
                  <p>既然要做 SVM，那么我们得想清楚要分清两个类别需要哪些特征。既然是特征，那我们就要选出二者不同的特征，这样更加有区分度。 比如这里我大体总结了有这么几个特征：</p>
                  <ul>
                    <li>文本密度：正文页通常会包含密集的文字，比如一个 p 节点内部就包含几十上百个文字，如果用单个节点内的文字数目来表示文本密度的话，那么详情页的部分内容文本密度会很高。</li>
                    <li>超链接节点的数量和比例：一般来说列表页通常会包含多个超链接，而且很大比例都是超链接文本，而详情页却有很多的文字并不是超链接，比如正文。</li>
                    <li>符号密度：一般来说列表页通常会是一些标题导航，一般可能都不会包含句号，而详情页的正文内容通常就会包含句号等内容，如果按照单位文字所包含的标点符号数量来表示符号密度的话，后者的符号密度也会高一些。</li>
                    <li>列表簇的数目：一般来说，列表页通常会包含多个具有公共父节点的条目，多个条目构成一个列表簇，虽然说详情页侧栏也会包含一些列表，但至少这个数量也可以成为一个特征来判别。</li>
                    <li>meta 信息：有一些特殊的 meta 信息是列表页独有的，比如只有详情页才会有发布时间，而列表页通常是没有的。</li>
                    <li>正文标题和 title 标题相似度：一般来说，详情页的正文标题和 title 标题很可能是相同的内容，而列表页通常则是站点的名称。</li>
                  </ul>
                  <p>以上便是我简单列的几个特征，还有很多其他的特征也可以来挖掘，比如视觉特征等等。</p>
                  <h2 id="模型训练"><a href="#模型训练" class="headerlink" title="模型训练"></a>模型训练</h2>
                  <p>真正代码实现的过程就是将现有的 HTML 文本进行预处理，把上面的一些特征提取出来，然后直接声明一个 SVM 分类模型即可。 这里声明了一个 feature 名字和对应的处理方法：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">self</span>.feature_funcs = &#123;</span><br><span class="line">    <span class="string">'number_of_a_char'</span>: number_of_a_char,</span><br><span class="line">    <span class="string">'number_of_a_char_log10'</span>: <span class="keyword">self</span>._number_of_a_char_log1<span class="number">0</span>,</span><br><span class="line">    <span class="string">'number_of_char'</span>: number_of_char,</span><br><span class="line">    <span class="string">'number_of_char_log10'</span>: <span class="keyword">self</span>._number_of_char_log1<span class="number">0</span>,</span><br><span class="line">    <span class="string">'rate_of_a_char'</span>: <span class="keyword">self</span>._rate_of_a_char,</span><br><span class="line">    <span class="string">'number_of_p_descendants'</span>: number_of_p_descendants,</span><br><span class="line">    <span class="string">'number_of_a_descendants'</span>: number_of_a_descendants,</span><br><span class="line">    <span class="string">'number_of_punctuation'</span>: number_of_punctuation,</span><br><span class="line">    <span class="string">'density_of_punctuation'</span>: density_of_punctuation,</span><br><span class="line">    <span class="string">'number_of_clusters'</span>: <span class="keyword">self</span>._number_of_clusters,</span><br><span class="line">    <span class="string">'density_of_text'</span>: density_of_text,</span><br><span class="line">    <span class="string">'max_density_of_text'</span>: <span class="keyword">self</span>._max_density_of_text,</span><br><span class="line">    <span class="string">'max_number_of_p_children'</span>: <span class="keyword">self</span>._max_number_of_p_children,</span><br><span class="line">    <span class="string">'has_datetime_meta'</span>: <span class="keyword">self</span>._has_datetime_mata,</span><br><span class="line">    <span class="string">'similarity_of_title'</span>: <span class="keyword">self</span>._similarity_of_title,</span><br><span class="line">&#125;</span><br><span class="line"><span class="keyword">self</span>.feature_names = <span class="keyword">self</span>.feature_funcs.keys()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上方法就是特征和对应的获取方法，具体根据实际情况实现即可。 然后关键的部分就是对数据的处理和模型的训练了，关键代码如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">list_file_paths = <span class="keyword">list</span>(<span class="built_in">glob</span>(<span class="keyword">f</span><span class="string">'&#123;DATASETS_LIST_DIR&#125;/*.html'</span>))</span><br><span class="line">detail_file_paths = <span class="keyword">list</span>(<span class="built_in">glob</span>(<span class="keyword">f</span><span class="string">'&#123;DATASETS_DETAIL_DIR&#125;/*.html'</span>))</span><br><span class="line"></span><br><span class="line">x_data, y_data = [], []</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> <span class="built_in">index</span>, list_file_path in enumerate(list_file_paths):</span><br><span class="line">    logger.<span class="built_in">log</span>(<span class="string">'inspect'</span>, <span class="keyword">f</span><span class="string">'list_file_path &#123;list_file_path&#125;'</span>)</span><br><span class="line">    element = file2element(list_file_path)</span><br><span class="line">    <span class="keyword">if</span> element <span class="keyword">is</span> None:</span><br><span class="line">        <span class="keyword">continue</span></span><br><span class="line">    preprocess4list_classifier(element)</span><br><span class="line">    <span class="keyword">x</span> = self.features_to_list(self.features(element))</span><br><span class="line">    x_data.<span class="keyword">append</span>(<span class="keyword">x</span>)</span><br><span class="line">    y_data.<span class="keyword">append</span>(<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> <span class="built_in">index</span>, detail_file_path in enumerate(detail_file_paths):</span><br><span class="line">    logger.<span class="built_in">log</span>(<span class="string">'inspect'</span>, <span class="keyword">f</span><span class="string">'detail_file_path &#123;detail_file_path&#125;'</span>)</span><br><span class="line">    element = file2element(detail_file_path)</span><br><span class="line">    <span class="keyword">if</span> element <span class="keyword">is</span> None:</span><br><span class="line">        <span class="keyword">continue</span></span><br><span class="line">    preprocess4list_classifier(element)</span><br><span class="line">    <span class="keyword">x</span> = self.features_to_list(self.features(element))</span><br><span class="line">    x_data.<span class="keyword">append</span>(<span class="keyword">x</span>)</span><br><span class="line">    y_data.<span class="keyword">append</span>(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line"># preprocess data</span><br><span class="line">ss = StandardScaler()</span><br><span class="line">x_data = ss.fit_transform(x_data)</span><br><span class="line">joblib.dump(ss, self.scaler_path)</span><br><span class="line">x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=<span class="number">0.2</span>, random_state=<span class="number">5</span>)</span><br><span class="line"></span><br><span class="line"># <span class="keyword">set</span> <span class="keyword">up</span> grid <span class="built_in">search</span></span><br><span class="line">c_range = np.logspace(-<span class="number">5</span>, <span class="number">20</span>, <span class="number">5</span>, base=<span class="number">2</span>)</span><br><span class="line">gamma_range = np.logspace(-<span class="number">9</span>, <span class="number">10</span>, <span class="number">5</span>, base=<span class="number">2</span>)</span><br><span class="line">param_grid = [</span><br><span class="line">    &#123;<span class="string">'kernel'</span>: [<span class="string">'rbf'</span>], <span class="string">'C'</span>: c_range, <span class="string">'gamma'</span>: gamma_range&#125;,</span><br><span class="line">    &#123;<span class="string">'kernel'</span>: [<span class="string">'linear'</span>], <span class="string">'C'</span>: c_range&#125;,</span><br><span class="line">]</span><br><span class="line">grid = GridSearchCV(SVC(probability=True), param_grid, cv=<span class="number">5</span>, <span class="keyword">verbose</span>=<span class="number">10</span>, n_jobs=-<span class="number">1</span>)</span><br><span class="line">clf = grid.fit(x_train, y_train)</span><br><span class="line">y_true, y_pred = y_test, clf.predict(x_test)</span><br><span class="line">logger.<span class="built_in">log</span>(<span class="string">'inspect'</span>, <span class="keyword">f</span><span class="string">'n&#123;classification_report(y_true, y_pred)&#125;'</span>)</span><br><span class="line">score = grid.score(x_test, y_test)</span><br><span class="line">logger.<span class="built_in">log</span>(<span class="string">'inspect'</span>, <span class="keyword">f</span><span class="string">'test accuracy &#123;score&#125;'</span>)</span><br><span class="line"># save model</span><br><span class="line">joblib.dump(grid.best_estimator_, self.model_path)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先对数据进行预处理，然后将每个 feature 存 存到 x_data 中，标注结果存到 y_data 中。接着我们使用 StandardScaler 对数据进行标准化处理，然后进行随机切分。最后使用 GridSearch 训练了一个 SVM 模型然后保存了下来。 以上便是基本的模型训练过程，具体的代码可以再完善一下。</p>
                  <h2 id="使用"><a href="#使用" class="headerlink" title="使用"></a>使用</h2>
                  <p>以上的流程我已经实现了，并且发布了一个开源 Python 包，名字叫做 Gerapy AutoExtractor，GitHub 地址为 <a href="https://github.com/Gerapy/GerapyAutoExtractor" target="_blank" rel="noopener">https://github.com/Gerapy/GerapyAutoExtractor</a>。 大家如需使用可以使用 pip 安装：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> gerapy-<span class="keyword">auto</span>-extractor</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个库针对于以上算法提供了四个方法：</p>
                  <ul>
                    <li>is_detail：判断是否是详情页</li>
                    <li>is_list：判断是否是列表页</li>
                    <li>probability_of_detail：是详情页的概率，结果是 0-1</li>
                    <li>probability_of_list：是列表页的概率，结果是 0-1</li>
                  </ul>
                  <p>例如，我们随便可以找几个网址存下来，比如把上文的列表页和详情页的 HTML 代码存下来分别保存为 list.html 和 detail.html。 测试代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from gerapy_auto_extractor import is_detail, is_list, probability_of_detail, probability_of_list</span><br><span class="line">from gerapy_auto_extractor<span class="selector-class">.helpers</span> import <span class="attribute">content</span>, jsonify</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = <span class="attribute">content</span>(<span class="string">'detail.html'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(probability_of_detail(html)</span></span>, probability_of_list(html))</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(is_detail(html)</span></span>, is_list(html))</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = <span class="attribute">content</span>(<span class="string">'list.html'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(probability_of_detail(html)</span></span>, probability_of_list(html))</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(is_detail(html)</span></span>, is_list(html))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们就调用了以上四个方法来实现了页面类型和置信度的判断。 类似的输出结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">0.9990605314033392</span> <span class="number">0.0009394685966607814</span></span><br><span class="line"><span class="literal">True</span> <span class="literal">False</span></span><br><span class="line"><span class="number">0.033477426883441685</span> <span class="number">0.9665225731165583</span></span><br><span class="line"><span class="literal">False</span> <span class="literal">True</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就成功得到了两个页面的类别和置信度了。 以上便是判断列表页和详情页的原理和实现，如需了解更多请关注项目 Gerapy Auto Extractor，GitHub 链接为 <a href="https://github.com/Gerapy/GerapyAutoExtractor，多谢支持" target="_blank" rel="noopener">https://github.com/Gerapy/GerapyAutoExtractor，多谢支持</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-02 17:19:51" itemprop="dateCreated datePublished" datetime="2020-08-02T17:19:51+08:00">2020-08-02</time>
                </span>
                <span id="/9531.html" class="post-meta-item leancloud_visitors" data-flag-title="如何区分一个页面是列表页还是详情页" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9529.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9529.html" class="post-title-link" itemprop="url">Airtest 和 Poco 的 API 总结</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="Airtest"><a href="#Airtest" class="headerlink" title="Airtest"></a>Airtest</h2>
                  <p>首先需要安装 Airtest，使用 pip3 即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> airtest</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="初始化-device"><a href="#初始化-device" class="headerlink" title="初始化 device"></a>初始化 device</h3>
                  <p>如果设备没有被初始化的话会进行初始化，并把初始化的设备作为当前设备。 用法如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">init_device</span><span class="params">(platform=<span class="string">"Android"</span>, uuid=None, **kwargs)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    Initialize device if not yet, and set as current device.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    :param platform: Android, IOS or Windows</span></span><br><span class="line"><span class="string">    :param uuid: uuid for target device, e.g. serialno for Android, handle for Windows, uuid for iOS</span></span><br><span class="line"><span class="string">    :param kwargs: Optional platform specific keyword args, e.g. `cap_method=JAVACAP` for Android</span></span><br><span class="line"><span class="string">    :return: device instance</span></span><br><span class="line"><span class="string">    """</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">device</span> = <span class="function"><span class="title">init_device</span>(<span class="string">'Android'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">device</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;airtest<span class="selector-class">.core</span><span class="selector-class">.android</span><span class="selector-class">.android</span><span class="selector-class">.Android</span> <span class="selector-tag">object</span> at <span class="number">0</span>x1018f3a58&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现它返回的是一个 Android 对象。 这个 Android 对象实际上属于 <code>airtest.core.android</code> 这个包，继承自 <code>airtest.core.device.Device</code> 这个类，与之并列的还有 <code>airtest.core.ios.ios.IOS</code> 、<code>airtest.core.linux.linux.Linux</code>、<code>airtest.core.win.win.Windows</code> 等。这些都有一些针对 Device 操作的 API，下面我们以 <code>airtest.core.android.android.Android</code> 为例来总结一下。</p>
                  <ul>
                    <li>get_default_device：获取默认 device</li>
                    <li>uuid：获取当前 Device 的 UUID</li>
                    <li>list_app：列举所有 App</li>
                    <li>path_app：打印输出某个 App 的完整路径</li>
                    <li>check_app：检查某个 App 是否在当前设备上</li>
                    <li>start_app：启动某个 App</li>
                    <li>start_app_timing：启动某个 App，然后计算时间</li>
                    <li>stop_app：停止某个 App</li>
                    <li>clear_app：清空某个 App 的全部数据</li>
                    <li>install_app：安装某个 App</li>
                    <li>install_multiple_app：安装多个 App</li>
                    <li>uninstall_app：卸载某个 App</li>
                    <li>snapshot：屏幕截图</li>
                    <li>shell：获取 Adb Shell 执行的结果</li>
                    <li>keyevent：执行键盘操作</li>
                    <li>wake：唤醒当前设备</li>
                    <li>home：点击 HOME 键</li>
                    <li>text：向设备输入内容</li>
                    <li>touch：点击屏幕某处的位置</li>
                    <li>double_click：双击屏幕某处的位置</li>
                    <li>swipe：滑动屏幕，由一点到另外一点</li>
                    <li>pinch：手指捏和操作</li>
                    <li>logcat：日志记录操作</li>
                    <li>getprop：获取某个特定属性的值</li>
                    <li>get_ip_address：获取 IP 地址</li>
                    <li>get_top_activity：获取当前 Activity</li>
                    <li>get_top_activity_name_and_pid：获取当前 Activity 的名称和进程号</li>
                    <li>get_top_activity_name：获取当前 Activity 的名称</li>
                    <li>is_keyboard_shown：判断当前键盘是否出现了</li>
                    <li>is_locked：设备是否锁定了</li>
                    <li>unlock：解锁设备</li>
                    <li>display_info：获取当前显示信息，如屏幕宽高等</li>
                    <li>get_display_info：同 display_info</li>
                    <li>get_current_resolution：获取当前设备分辨率</li>
                    <li>get_render_resolution：获取当前渲染分辨率</li>
                    <li>start_recording：开始录制</li>
                    <li>stop_recording：结束录制</li>
                    <li>adjust_all_screen：调整屏幕适配分辨率</li>
                  </ul>
                  <p>了解了上面的方法之后，我们可以用一个实例来感受下它们的用法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> airtest.core.android import Android</span><br><span class="line"><span class="keyword">from</span> airtest.core.api import *</span><br><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logging.getLogger(<span class="string">"airtest"</span>).setLevel(logging.WARNING)</span><br><span class="line"></span><br><span class="line">device: Android = init_device(<span class="string">'Android'</span>)</span><br><span class="line">is_locked = device.is_locked()</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'is_locked: &#123;is_locked&#125;'</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> is_locked:</span><br><span class="line">    device.unlock()</span><br><span class="line"></span><br><span class="line">device.wake()</span><br><span class="line"></span><br><span class="line">app_list = device.list_app()</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'app list &#123;app_list&#125;'</span>)</span><br><span class="line"></span><br><span class="line">uuid = device.uuid</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'uuid &#123;uuid&#125;'</span>)</span><br><span class="line"></span><br><span class="line">display_info = device.get_display_info()</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'display info &#123;display_info&#125;'</span>)</span><br><span class="line"></span><br><span class="line">resolution = device.get_render_resolution()</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'resolution &#123;resolution&#125;'</span>)</span><br><span class="line"></span><br><span class="line">ip_address = device.get_ip_address()</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'ip address &#123;ip_address&#125;'</span>)</span><br><span class="line"></span><br><span class="line">top_activity = device.get_top_activity()</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'top activity &#123;top_activity&#125;'</span>)</span><br><span class="line"></span><br><span class="line">is_keyboard_shown = device.is_keyboard_shown()</span><br><span class="line"><span class="builtin-name">print</span>(f<span class="string">'is keyboard shown &#123;is_keyboard_shown&#125;'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们调用了设备的一些操作方法，获取了一些基本状态，运行结果如下：</p>
                  <figure class="highlight sml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">is_locked: <span class="type">False</span></span><br><span class="line">app <span class="built_in">list</span> [<span class="symbol">'com</span>.kimcy929.screenrecorder', <span class="symbol">'com</span>.android.providers.telephony', <span class="symbol">'io</span>.appium.settings', <span class="symbol">'com</span>.android.providers.calendar', <span class="symbol">'com</span>.android.providers.media', <span class="symbol">'com</span>.goldze.mvvmhabit', <span class="symbol">'com</span>.android.wallpapercropper', <span class="symbol">'com</span>.android.documentsui', <span class="symbol">'com</span>.android.galaxy4', <span class="symbol">'com</span>.android.externalstorage', <span class="symbol">'com</span>.android.htmlviewer', <span class="symbol">'com</span>.android.quicksearchbox', <span class="symbol">'com</span>.android.mms.service', <span class="symbol">'com</span>.android.providers.downloads', <span class="symbol">'mark</span>.qrcode', ..., <span class="symbol">'com</span>.google.android.play.games', <span class="symbol">'io</span>.kkzs', <span class="symbol">'tv</span>.danmaku.bili', <span class="symbol">'com</span>.android.captiveportallogin']</span><br><span class="line">uuid emulator-<span class="number">5554</span></span><br><span class="line">display info &#123;<span class="symbol">'id'</span>: <span class="number">0</span>, <span class="symbol">'width'</span>: <span class="number">1080</span>, <span class="symbol">'height'</span>: <span class="number">1920</span>, <span class="symbol">'xdpi'</span>: <span class="number">320.0</span>, <span class="symbol">'ydpi'</span>: <span class="number">320.0</span>, <span class="symbol">'size'</span>: <span class="number">6.88</span>, <span class="symbol">'density'</span>: <span class="number">2.0</span>, <span class="symbol">'fps'</span>: <span class="number">60.0</span>, <span class="symbol">'secure'</span>: <span class="type">True</span>, <span class="symbol">'rotation'</span>: <span class="number">0</span>, <span class="symbol">'orientation'</span>: <span class="number">0.0</span>, <span class="symbol">'physical_width'</span>: <span class="number">1080</span>, <span class="symbol">'physical_height'</span>: <span class="number">1920</span>&#125;</span><br><span class="line">resolution (<span class="number">0.0</span>, <span class="number">0.0</span>, <span class="number">1080.0</span>, <span class="number">1920.0</span>)</span><br><span class="line">ip address <span class="number">10.0</span>.<span class="number">2.15</span></span><br><span class="line">top activity (<span class="symbol">'com</span>.microsoft.launcher.dev', <span class="symbol">'com</span>.microsoft.launcher.<span class="type">Launcher'</span>, <span class="string">'16040'</span>)</span><br><span class="line">is keyboard shown <span class="type">False</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="连接-device"><a href="#连接-device" class="headerlink" title="连接 device"></a>连接 device</h3>
                  <p>连接 device 需要传入设备的 uri，格式类似 <code>android://adbhost:adbport/serialno?param=value&amp;param2=value2</code>，使用方法如下：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def connect_device(uri):</span><br><span class="line">    """</span><br><span class="line">    Initialize device <span class="keyword">with</span> uri, <span class="keyword">and</span> <span class="keyword">set</span> <span class="keyword">as</span> <span class="keyword">current</span> device.</span><br><span class="line"></span><br><span class="line">    :param uri: an URI <span class="keyword">where</span> <span class="keyword">to</span> <span class="keyword">connect</span> <span class="keyword">to</span> device, e.g. <span class="string">`android://adbhost:adbport/serialno?param=value&amp;param2=value2`</span></span><br><span class="line">    :<span class="keyword">return</span>: device <span class="keyword">instance</span></span><br><span class="line">    :Example:</span><br><span class="line">        * <span class="string">``</span>android:///<span class="string">``</span> <span class="comment"># local adb device using default params</span></span><br><span class="line">        * <span class="string">``</span>android://adbhost:adbport/<span class="number">1234566</span>?cap_method=javacap&amp;touch_method=adb<span class="string">``</span>  <span class="comment"># remote device using custom params</span></span><br><span class="line">        * <span class="string">``</span>windows:///<span class="string">``</span> <span class="comment"># local Windows application</span></span><br><span class="line">        * <span class="string">``</span>ios:///<span class="string">``</span> <span class="comment"># iOS device</span></span><br><span class="line">    <span class="string">"""</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例如下：</p>
                  <figure class="highlight coffeescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> airtest.core.android <span class="keyword">import</span> Android</span><br><span class="line"><span class="keyword">from</span> airtest.core.api <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line">uri = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line">device: Android = connect_device(uri)</span><br><span class="line"><span class="built_in">print</span>(device)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;airtest<span class="selector-class">.core</span><span class="selector-class">.android</span><span class="selector-class">.android</span><span class="selector-class">.Android</span> <span class="selector-tag">object</span> at <span class="number">0</span>x110246940&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其实返回结果和 init_device 是一样的，最后 connect_device 方法就是调用了 init_device 方法。</p>
                  <h3 id="获取当前-device"><a href="#获取当前-device" class="headerlink" title="获取当前 device"></a>获取当前 device</h3>
                  <p>就是直接调用 device 方法，定义如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">device</span><span class="params">()</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    Return the current active device.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    :return: current device instance</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    <span class="keyword">return</span> G.DEVICE</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="获取所有-device"><a href="#获取所有-device" class="headerlink" title="获取所有 device"></a>获取所有 device</h3>
                  <p>在 airtest 中有一个全局变量 G，获取所有 device 的方法如下：</p>
                  <figure class="highlight coffeescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> airtest.core.android <span class="keyword">import</span> Android</span><br><span class="line"><span class="keyword">from</span> airtest.core.api <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(G.DEVICE_LIST)</span><br><span class="line">uri = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line">device: Android = connect_device(uri)</span><br><span class="line"><span class="built_in">print</span>(G.DEVICE_LIST)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[]</span><br><span class="line"><span class="meta">[&lt;airtest.core.android.android.Android object at 0x10ba03978&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里需要注意的是，在最开始没有调用 connect_device 方法之前，DEVICE_LIST 是空的，在调用之后 DEVICE_LIST 会自动添加已经连接的 device，DEVICE_LIST 就是已经连接的 device 列表</p>
                  <h3 id="切换-device"><a href="#切换-device" class="headerlink" title="切换 device"></a>切换 device</h3>
                  <p>我们可以使用 set_current 方法切换当前连接的 device，传入的是 index，定义如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">set_current</span><span class="params">(idx)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    Set current active device.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    :param idx: uuid or index of initialized device instance</span></span><br><span class="line"><span class="string">    :raise IndexError: raised when device idx is not found</span></span><br><span class="line"><span class="string">    :return: None</span></span><br><span class="line"><span class="string">    :platforms: Android, iOS, Windows</span></span><br><span class="line"><span class="string">    """</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个方法没有返回值，调用 set_current 方法切换 device 之后，再调用 device 方法就可以获取当前 device 对象了。</p>
                  <h3 id="执行命令行"><a href="#执行命令行" class="headerlink" title="执行命令行"></a>执行命令行</h3>
                  <p>可以使用 shell 方法传入 cmd 来执行命令行，定义如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">@logwrap</span><br><span class="line">def <span class="keyword">shell</span>(cmd):</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br><span class="line">    Start remote <span class="keyword">shell</span> in the target device <span class="built_in">and</span> <span class="keyword">execute</span> the <span class="keyword">command</span></span><br><span class="line"></span><br><span class="line">    :param cmd: <span class="keyword">command</span> <span class="keyword">to</span> <span class="keyword">be</span> run <span class="keyword">on</span> device, <span class="keyword">e</span>.g. <span class="string">"ls /data/local/tmp"</span></span><br><span class="line">    :<span class="keyword">return</span>: the output of the <span class="keyword">shell</span> cmd</span><br><span class="line">    :platform<span class="variable">s:</span> Android</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br><span class="line">    <span class="keyword">return</span> G.DEVICE.<span class="keyword">shell</span>(cmd)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>直接调用 adb 命令就好了，例如获取内存信息就可以使用如下命令：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">airtest.core.api</span> <span class="variable">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="variable">uri</span> = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line"><span class="function"><span class="title">connect_device</span>(<span class="variable">uri</span>)</span></span><br><span class="line"></span><br><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="function"><span class="title">shell</span>(<span class="string">'cat /proc/meminfo'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">MemTotal:</span>        <span class="number">3627908</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">MemFree:</span>         <span class="number">2655560</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">MemAvailable:</span>    <span class="number">2725928</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Buffers:</span>            <span class="number">3496</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Cached:</span>           <span class="number">147472</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">SwapCached:</span>            <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Active:</span>           <span class="number">744592</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Inactive:</span>         <span class="number">126332</span> <span class="string">kB</span></span><br><span class="line"><span class="string">Active(anon):</span>     <span class="number">723292</span> <span class="string">kB</span></span><br><span class="line"><span class="string">Inactive(anon):</span>    <span class="number">16344</span> <span class="string">kB</span></span><br><span class="line"><span class="string">Active(file):</span>      <span class="number">21300</span> <span class="string">kB</span></span><br><span class="line"><span class="string">Inactive(file):</span>   <span class="number">109988</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Unevictable:</span>           <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Mlocked:</span>               <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">HighTotal:</span>       <span class="number">2760648</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">HighFree:</span>        <span class="number">2073440</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">LowTotal:</span>         <span class="number">867260</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">LowFree:</span>          <span class="number">582120</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">SwapTotal:</span>             <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">SwapFree:</span>              <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Dirty:</span>                 <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Writeback:</span>             <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">AnonPages:</span>        <span class="number">720100</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Mapped:</span>           <span class="number">127720</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Shmem:</span>             <span class="number">19428</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Slab:</span>              <span class="number">76196</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">SReclaimable:</span>       <span class="number">7392</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">SUnreclaim:</span>        <span class="number">68804</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">KernelStack:</span>        <span class="number">7896</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">PageTables:</span>         <span class="number">8544</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">NFS_Unstable:</span>          <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Bounce:</span>                <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">WritebackTmp:</span>          <span class="number">0</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">CommitLimit:</span>     <span class="number">1813952</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">Committed_AS:</span>   <span class="number">21521776</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">VmallocTotal:</span>     <span class="number">122880</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">VmallocUsed:</span>       <span class="number">38876</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">VmallocChunk:</span>      <span class="number">15068</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">DirectMap4k:</span>       <span class="number">16376</span> <span class="string">kB</span></span><br><span class="line"><span class="attr">DirectMap4M:</span>      <span class="number">892928</span> <span class="string">kB</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="启动和停止-App"><a href="#启动和停止-App" class="headerlink" title="启动和停止 App"></a>启动和停止 App</h3>
                  <p>启动和停止 App 直接传入包名即可，其实它们就是调用的 device 的 start_app 和 stop_app 方法，定义如下：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def start_app(package, activity=None):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Start the target application on device</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">package</span>: name of the package to be started, e.g. <span class="string">"com.netease.my"</span></span><br><span class="line">    :param <span class="attribute">activity</span>: the activity to start, default is None which means the main activity</span><br><span class="line">    :<span class="attribute">return</span>: None</span><br><span class="line">    :<span class="attribute">platforms</span>: Android, iOS</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    G.DEVICE.start_app(package, activity)</span><br><span class="line"></span><br><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def stop_app(package):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Stop the target application on device</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">package</span>: name of the package to stop, see also <span class="built_in">`start_app`</span></span><br><span class="line">    :<span class="attribute">return</span>: None</span><br><span class="line">    :<span class="attribute">platforms</span>: Android, iOS</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    G.DEVICE.stop_app(package)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>用法示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">airtest.core.api</span> <span class="variable">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="variable">uri</span> = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line"><span class="function"><span class="title">connect_device</span>(<span class="variable">uri</span>)</span></span><br><span class="line"></span><br><span class="line"><span class="variable">package</span> = <span class="string">'com.tencent.mm'</span></span><br><span class="line"><span class="function"><span class="title">start_app</span>(<span class="variable">package</span>)</span></span><br><span class="line"><span class="function"><span class="title"><span class="built_in">sleep</span></span>(<span class="number">10</span>)</span></span><br><span class="line"><span class="function"><span class="title">stop_app</span>(<span class="variable">package</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我指定了微信的包名，然后调用 start_app 启动了微信，然后等待了 10 秒，然后调用了 stop_app 停止了微信。</p>
                  <h3 id="安装和卸载"><a href="#安装和卸载" class="headerlink" title="安装和卸载"></a>安装和卸载</h3>
                  <p>安装和卸载也是一样，也是调用了 device 的 install 和 uninstall 方法，定义如下：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def install(filepath, **kwargs):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Install application on device</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">filepath</span>: the path to file to be installed on target device</span><br><span class="line">    :param <span class="attribute">kwargs</span>: platform specific <span class="built_in">`kwargs`</span>, please refer to corresponding docs</span><br><span class="line">    :<span class="attribute">return</span>: None</span><br><span class="line">    :<span class="attribute">platforms</span>: Android</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    return G.DEVICE.install_app(filepath, **kwargs)</span><br><span class="line"></span><br><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def uninstall(package):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Uninstall application on device</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">package</span>: name of the package, see also <span class="built_in">`start_app`</span></span><br><span class="line">    :<span class="attribute">return</span>: None</span><br><span class="line">    :<span class="attribute">platforms</span>: Android</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    return G.DEVICE.uninstall_app(package)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="截图"><a href="#截图" class="headerlink" title="截图"></a>截图</h3>
                  <p>截图使用 snapshot 即可完成，可以设定存储的文件名称，图片质量等。 定义如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def snapshot(filename=None, msg=<span class="string">""</span>, quality=ST.SNAPSHOT_QUALITY):</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br><span class="line">    Take the screenshot of the target device <span class="built_in">and</span> save it <span class="keyword">to</span> the <span class="keyword">file</span>.</span><br><span class="line"></span><br><span class="line">    :param filename: name of the <span class="keyword">file</span> where <span class="keyword">to</span> save the screenshot. If the relative path <span class="keyword">is</span> provided, the default</span><br><span class="line">                     location <span class="keyword">is</span> ``ST.LOG_DIR``</span><br><span class="line">    :param ms<span class="variable">g:</span> short description <span class="keyword">for</span> screenshot, it will <span class="keyword">be</span> recorded in the report</span><br><span class="line">    :param quality: The image quality, integer in <span class="built_in">range</span> [<span class="number">1</span>, <span class="number">99</span>]</span><br><span class="line">    :<span class="keyword">return</span>: absolute path of the screenshot</span><br><span class="line">    :platform<span class="variable">s:</span> Android, iOS, Windows</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例如下：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> airtest.core.api <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line">uri = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line">connect_device(uri)</span><br><span class="line"></span><br><span class="line"><span class="keyword">package</span> = <span class="string">'com.tencent.mm'</span></span><br><span class="line">start_app(<span class="keyword">package</span>)</span><br><span class="line">sleep(<span class="number">3</span>)</span><br><span class="line">snapshot(<span class="string">'weixin.png'</span>, quality=<span class="number">30</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后在当前目录会生成一个 weixin.png 的图片，如图所示： <img src="https://qiniu.cuiqingcai.com/24t0f.png" alt="weixin"></p>
                  <h3 id="唤醒和首页"><a href="#唤醒和首页" class="headerlink" title="唤醒和首页"></a>唤醒和首页</h3>
                  <p>唤醒和回到首页分别也是调用了 device 的 wake 和 home 方法，定义如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">@logwrap</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">wake</span><span class="params">()</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    Wake up and unlock the target device</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    :return: None</span></span><br><span class="line"><span class="string">    :platforms: Android</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    .. note:: Might not work on some models</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    G.DEVICE.wake()</span><br><span class="line"></span><br><span class="line"><span class="meta">@logwrap</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">home</span><span class="params">()</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    Return to the home screen of the target device.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    :return: None</span></span><br><span class="line"><span class="string">    :platforms: Android, iOS</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    G.DEVICE.home()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>直接调用即可。</p>
                  <h3 id="点击屏幕"><a href="#点击屏幕" class="headerlink" title="点击屏幕"></a>点击屏幕</h3>
                  <p>点击屏幕是 touch 方法，可以传入一张图或者绝对位置，同时可以指定点击次数，定义如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">@logwrap</span><br><span class="line">def touch(v, times=<span class="number">1</span>, **kwargs):</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br><span class="line">    Perform the touch action <span class="keyword">on</span> the device screen</span><br><span class="line"></span><br><span class="line">    :param <span class="variable">v:</span> target <span class="keyword">to</span> touch, either <span class="keyword">a</span> Template instance <span class="built_in">or</span> absolute coordinates (<span class="keyword">x</span>, <span class="keyword">y</span>)</span><br><span class="line">    :param time<span class="variable">s:</span> how many touches <span class="keyword">to</span> <span class="keyword">be</span> performed</span><br><span class="line">    :param kwarg<span class="variable">s:</span> platform specific `kwargs`, please refer <span class="keyword">to</span> corresponding docs</span><br><span class="line">    :<span class="keyword">return</span>: finial position <span class="keyword">to</span> <span class="keyword">be</span> clicked</span><br><span class="line">    :platform<span class="variable">s:</span> Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如我现在的手机屏幕是这样子： <img src="https://qiniu.cuiqingcai.com/rn19x.png" alt="image-20200723104955681"> 这里我截图下来一张图片，如图所示： <img src="https://qiniu.cuiqingcai.com/sy0u5.png" alt="tpl"> 然后我们把这个图片声明成一个 Template 传入，示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">airtest.core.api</span> <span class="variable">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="variable">uri</span> = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line"><span class="function"><span class="title">connect_device</span>(<span class="variable">uri</span>)</span></span><br><span class="line"><span class="function"><span class="title">touch</span>(<span class="title">Template</span>(<span class="string">'tpl.png'</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>启动之后它就会识别出这张图片的位置，然后点击。 或者我们可以指定点击的绝对位置，示例如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> airtest.core.api <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line">uri = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line">connect_device(uri)</span><br><span class="line">home()</span><br><span class="line">touch((<span class="number">400</span>, <span class="number">600</span>), times=<span class="number">2</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外上述的 touch 方法还可以完全等同于 click 方法。 如果要双击的话，还可以使用调用 double_click 方法，传入参数也可以是 Template 或者绝对位置。</p>
                  <h3 id="滑动"><a href="#滑动" class="headerlink" title="滑动"></a>滑动</h3>
                  <p>滑动可以使用 swipe 方法，可以传入起始和终止位置，两个位置都可以传入绝对位置或者 Template，定义如下：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def swipe(v1, v2=None, vector=None, **kwargs):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Perform the swipe action on the device screen.</span><br><span class="line"></span><br><span class="line">    There are two ways of assigning the parameters</span><br><span class="line">        * <span class="built_in">``</span>swipe(v1, v2=Template(...))<span class="built_in">``</span>   # swipe from v1 to v2</span><br><span class="line">        * <span class="built_in">``</span>swipe(v1, vector=(x, y))<span class="built_in">``</span>      # swipe starts at v1 and moves along the vector.</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">v1</span>: the start point of swipe,</span><br><span class="line">               either a Template instance or absolute coordinates (x, y)</span><br><span class="line">    :param <span class="attribute">v2</span>: the end point of swipe,</span><br><span class="line">               either a Template instance or absolute coordinates (x, y)</span><br><span class="line">    :param <span class="attribute">vector</span>: a vector coordinates of swipe action, either absolute coordinates (x, y) or percentage of</span><br><span class="line">                   screen e.g.(<span class="number">0.5</span>, <span class="number">0.5</span>)</span><br><span class="line">    :param **<span class="attribute">kwargs</span>: platform specific <span class="built_in">`kwargs`</span>, please refer to corresponding docs</span><br><span class="line">    :raise <span class="attribute">Exception</span>: general exception when not enough parameters to perform swap action have been provided</span><br><span class="line">    :<span class="attribute">return</span>: Origin position and target position</span><br><span class="line">    :<span class="attribute">platforms</span>: Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span>"</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>比如这里我们可以定义手指向右滑动：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> airtest.core.api <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line">uri = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line">connect_device(uri)</span><br><span class="line">home()</span><br><span class="line">swipe((<span class="number">200</span>, <span class="number">300</span>), (<span class="number">900</span>, <span class="number">300</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="放大缩小"><a href="#放大缩小" class="headerlink" title="放大缩小"></a>放大缩小</h3>
                  <p>放大缩小是使用的 pinch 方法，可以指定放大还是缩小，同时还可以指定中心位置点和放大缩小的比率。 定义如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">@logwrap</span><br><span class="line">def pinch(in_or_out=<span class="string">'in'</span>, <span class="keyword">center</span>=None, percent=<span class="number">0.5</span>):</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br><span class="line">    Perform the pinch action <span class="keyword">on</span> the device screen</span><br><span class="line"></span><br><span class="line">    :param in_or_ou<span class="variable">t:</span> pinch in <span class="built_in">or</span> pinch out, enum in [<span class="string">"in"</span>, <span class="string">"out"</span>]</span><br><span class="line">    :param <span class="keyword">center</span>: <span class="keyword">center</span> of pinch action, default <span class="keyword">as</span> None which <span class="keyword">is</span> the <span class="keyword">center</span> of the screen</span><br><span class="line">    :param percen<span class="variable">t:</span> percentage of the screen of pinch action, default <span class="keyword">is</span> <span class="number">0.5</span></span><br><span class="line">    :<span class="keyword">return</span>: None</span><br><span class="line">    :platform<span class="variable">s:</span> Android</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> airtest.core.api import *</span><br><span class="line"></span><br><span class="line">uri = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line">connect_device(uri)</span><br><span class="line">home()</span><br><span class="line">pinch(<span class="attribute">in_or_out</span>=<span class="string">'out'</span>, center=(300, 300), <span class="attribute">percent</span>=0.4)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="键盘事件"><a href="#键盘事件" class="headerlink" title="键盘事件"></a>键盘事件</h3>
                  <p>可以使用 keyevent 来输入某个键，例如 home、back 等等，keyevent 的定义如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">keyevent</span><span class="params">(keyname, **kwargs)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    Perform key event on the device</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    :param keyname: platform specific key name</span></span><br><span class="line"><span class="string">    :param **kwargs: platform specific `kwargs`, please refer to corresponding docs</span></span><br><span class="line"><span class="string">    :return: None</span></span><br><span class="line"><span class="string">    :platforms: Android, Windows, iOS</span></span><br><span class="line"><span class="string">    """</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">keyevent</span><span class="params">(<span class="string">"HOME"</span>)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就表示按了 HOME 键。</p>
                  <h3 id="输入内容"><a href="#输入内容" class="headerlink" title="输入内容"></a>输入内容</h3>
                  <p>输入内容需要使用 text 方法，当然前提是这个 widget 需要是 active 状态，text 的定义如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">@logwrap</span><br><span class="line">def text(text, enter=True, **kwargs):</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br><span class="line">    Input text <span class="keyword">on</span> the target device. Text <span class="built_in">input</span> widget must <span class="keyword">be</span> active <span class="keyword">first</span>.</span><br><span class="line"></span><br><span class="line">    :param tex<span class="variable">t:</span> text <span class="keyword">to</span> <span class="built_in">input</span>, unicode <span class="keyword">is</span> supported</span><br><span class="line">    :param enter: <span class="built_in">input</span> `Enter` keyevent after text <span class="built_in">input</span>, default <span class="keyword">is</span> True</span><br><span class="line">    :<span class="keyword">return</span>: None</span><br><span class="line">    :platform<span class="variable">s:</span> Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="等待和判断"><a href="#等待和判断" class="headerlink" title="等待和判断"></a>等待和判断</h3>
                  <p>可以使用 wait 方法等待某个内容加载出来，需要传入的是 Template，定义如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">@logwrap</span><br><span class="line">def wait(v, timeout=None, interval=<span class="number">0.5</span>, intervalfunc=None):</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br><span class="line">    Wait <span class="keyword">to</span> <span class="keyword">match</span> the Template <span class="keyword">on</span> the device screen</span><br><span class="line"></span><br><span class="line">    :param <span class="variable">v:</span> target object <span class="keyword">to</span> wait <span class="keyword">for</span>, Template instance</span><br><span class="line">    :param timeou<span class="variable">t:</span> time interval <span class="keyword">to</span> wait <span class="keyword">for</span> the <span class="keyword">match</span>, default <span class="keyword">is</span> None which <span class="keyword">is</span> ``ST.FIND_TIMEOUT``</span><br><span class="line">    :param interva<span class="variable">l:</span> time interval in seconds <span class="keyword">to</span> attempt <span class="keyword">to</span> <span class="keyword">find</span> <span class="keyword">a</span> <span class="keyword">match</span></span><br><span class="line">    :param intervalfunc: called after each unsuccessful attempt <span class="keyword">to</span> <span class="keyword">find</span> the corresponding <span class="keyword">match</span></span><br><span class="line">    :raise TargetNotFoundError: raised <span class="keyword">if</span> target <span class="keyword">is</span> not found after the time limit expired</span><br><span class="line">    :<span class="keyword">return</span>: coordinates of the matched target</span><br><span class="line">    :platform<span class="variable">s:</span> Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span><span class="comment">"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>同时也使用 exists 方法判断某个内容是否存在，定义如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">@logwrap</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">exists</span><span class="params">(v)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    Check whether given target exists on device screen</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    :param v: target to be checked</span></span><br><span class="line"><span class="string">    :return: False if target is not found, otherwise returns the coordinates of the target</span></span><br><span class="line"><span class="string">    :platforms: Android, Windows, iOS</span></span><br><span class="line"><span class="string">    """</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="断言"><a href="#断言" class="headerlink" title="断言"></a>断言</h3>
                  <p>另外 Airtest 还提供了几个断言语句来判断结果是否存在或者相同，定义如下：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def assert_exists(v, msg=<span class="string">""</span>):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Assert target exists on device screen</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">v</span>: target to be checked</span><br><span class="line">    :param <span class="attribute">msg</span>: short description of assertion, it will be recorded in the report</span><br><span class="line">    :raise <span class="attribute">AssertionError</span>: if assertion fails</span><br><span class="line">    :<span class="attribute">return</span>: coordinates of the target</span><br><span class="line">    :<span class="attribute">platforms</span>: Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    <span class="attribute">try</span>:</span><br><span class="line">        pos = loop_find(v, timeout=ST.FIND_TIMEOUT, threshold=ST.THRESHOLD_STRICT)</span><br><span class="line">        return pos</span><br><span class="line">    except <span class="attribute">TargetNotFoundError</span>:</span><br><span class="line">        raise AssertionError(<span class="string">"%s does not exist in screen, message: %s"</span> % (v, msg))</span><br><span class="line"></span><br><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def assert_not_exists(v, msg=<span class="string">""</span>):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Assert target does not exist on device screen</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">v</span>: target to be checked</span><br><span class="line">    :param <span class="attribute">msg</span>: short description of assertion, it will be recorded in the report</span><br><span class="line">    :raise <span class="attribute">AssertionError</span>: if assertion fails</span><br><span class="line">    :<span class="attribute">return</span>: None.</span><br><span class="line">    :<span class="attribute">platforms</span>: Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    <span class="attribute">try</span>:</span><br><span class="line">        pos = loop_find(v, timeout=ST.FIND_TIMEOUT_TMP)</span><br><span class="line">        raise AssertionError(<span class="string">"%s exists unexpectedly at pos: %s, message: %s"</span> % (v, pos, msg))</span><br><span class="line">    except <span class="attribute">TargetNotFoundError</span>:</span><br><span class="line">        pass</span><br><span class="line"></span><br><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def assert_equal(first, second, msg=<span class="string">""</span>):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Assert two values are equal</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">first</span>: first value</span><br><span class="line">    :param <span class="attribute">second</span>: second value</span><br><span class="line">    :param <span class="attribute">msg</span>: short description of assertion, it will be recorded in the report</span><br><span class="line">    :raise <span class="attribute">AssertionError</span>: if assertion fails</span><br><span class="line">    :<span class="attribute">return</span>: None</span><br><span class="line">    :<span class="attribute">platforms</span>: Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    if first != <span class="attribute">second</span>:</span><br><span class="line">        raise AssertionError(<span class="string">"%s and %s are not equal, message: %s"</span> % (first, second, msg))</span><br><span class="line"></span><br><span class="line"><span class="variable">@logwrap</span></span><br><span class="line">def assert_not_equal(first, second, msg=<span class="string">""</span>):</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    Assert two values are not equal</span><br><span class="line"></span><br><span class="line">    :param <span class="attribute">first</span>: first value</span><br><span class="line">    :param <span class="attribute">second</span>: second value</span><br><span class="line">    :param <span class="attribute">msg</span>: short description of assertion, it will be recorded in the report</span><br><span class="line">    :raise <span class="attribute">AssertionError</span>: if assertion</span><br><span class="line">    :<span class="attribute">return</span>: None</span><br><span class="line">    :<span class="attribute">platforms</span>: Android, Windows, iOS</span><br><span class="line">    <span class="string">""</span>"</span><br><span class="line">    if first == <span class="attribute">second</span>:</span><br><span class="line">        raise AssertionError(<span class="string">"%s and %s are equal, message: %s"</span> % (first, second, msg))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这几个断言比较常用的就是 assert_exists 和 assert_not_exists 判断某个目标是否存在于屏幕上，同时还可以传入 msg，它可以被记录到 report 里面。 以上就是 Airtest 的 API 的用法，它提供了一些便捷的方法封装，同时还对接了图像识别等技术。 但 Airtest 也有一定的局限性，比如不能根据 DOM 树来选择对应的节点，依靠图像识别也有一定的不精确之处，所以还需要另外一个库 —— Poco。</p>
                  <h2 id="Poco"><a href="#Poco" class="headerlink" title="Poco"></a>Poco</h2>
                  <p>利用 Poco 我们可以支持 DOM 选择，例如编写 XPath 等来定位某一个节点。 首先需要安装 Poco，使用 pip3 即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> pocoui</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装好了之后我们便可以使用它来选择一些节点了，示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">airtest.core.api</span> <span class="variable">import</span> *</span><br><span class="line"><span class="variable">from</span> <span class="variable">poco.drivers.android.uiautomation</span> <span class="variable">import</span> <span class="variable">AndroidUiautomationPoco</span></span><br><span class="line"></span><br><span class="line"><span class="variable">poco</span> = <span class="function"><span class="title">AndroidUiautomationPoco</span>()</span></span><br><span class="line"></span><br><span class="line"><span class="variable">uri</span> = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line"><span class="function"><span class="title">connect_device</span>(<span class="variable">uri</span>)</span></span><br><span class="line"><span class="function"><span class="title">home</span>()</span></span><br><span class="line"><span class="function"><span class="title">poco</span>(<span class="variable">text</span>=<span class="string">'Weather'</span>).click()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>比如这里我们就声明了 AndroidUiautomationPoco 这个 Poco 对象，然后调用了 poco 传入一些选择器，选中之后执行 click 方法。 这个选择器非常强大，可以传入 name 和各个属性值，具体的使用方法见：<a href="https://poco.readthedocs.io/en/latest/source/poco.pocofw.html" target="_blank" rel="noopener">https://poco.readthedocs.io/en/latest/source/poco.pocofw.html</a></p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">airtest.core.api</span> <span class="variable">import</span> *</span><br><span class="line"><span class="variable">from</span> <span class="variable">poco.drivers.android.uiautomation</span> <span class="variable">import</span> <span class="variable">AndroidUiautomationPoco</span></span><br><span class="line"><span class="variable">from</span> <span class="variable">poco.proxy</span> <span class="variable">import</span> <span class="variable">UIObjectProxy</span></span><br><span class="line"></span><br><span class="line"><span class="variable">poco</span> = <span class="function"><span class="title">AndroidUiautomationPoco</span>()</span></span><br><span class="line"></span><br><span class="line"><span class="variable">uri</span> = <span class="string">'Android://127.0.0.1:5037/emulator-5554'</span></span><br><span class="line"><span class="function"><span class="title">connect_device</span>(<span class="variable">uri</span>)</span></span><br><span class="line"><span class="function"><span class="title">home</span>()</span></span><br><span class="line"><span class="variable"><span class="class">object</span></span>: <span class="variable">UIObjectProxy</span> = <span class="function"><span class="title">poco</span>(<span class="string">"com.microsoft.launcher.dev:id/workspace"</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">object</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>poco 返回的是 UIObjectProxy 对象，它提供了其他的操作 API，例如选取子节点，兄弟节点，父节点等等，同时可以调用各个操作方法，如 click、pinch、scroll 等等。 具体的操作文档可以参见：<a href="https://poco.readthedocs.io/en/latest/source/poco.proxy.html" target="_blank" rel="noopener">https://poco.readthedocs.io/en/latest/source/poco.proxy.html</a> 下面简单总结：</p>
                  <ul>
                    <li>attr：获取节点属性</li>
                    <li>child：获取子节点</li>
                    <li>children：获取所有子节点</li>
                    <li>click：点击</li>
                    <li>double_click：双击</li>
                    <li>drag_to：将某个节点拖拽到另一个节点</li>
                    <li>exists：某个节点是否存在</li>
                    <li>focus：获得焦点</li>
                    <li>get_bounds：获取边界</li>
                    <li>get_name：获取节点名</li>
                    <li>get_position：获取节点位置</li>
                    <li>get_size：获取节点大小</li>
                    <li>get_text：获取文本内容</li>
                    <li>long_click：长按</li>
                    <li>offspring：选出包含直接子代的后代</li>
                    <li>parent：父节点</li>
                    <li>pinch：缩放</li>
                    <li>scroll：滑动</li>
                    <li>set_text：设置文字</li>
                    <li>sibling：兄弟节点</li>
                    <li>swipe：滑动</li>
                    <li>wait：等待</li>
                    <li>wait_for_appearance：等待某个节点出现</li>
                    <li>wait_for_disappearance：等待某个节点消失</li>
                  </ul>
                  <p>以上的这些方法混用的话就可以执行各种节点的选择和相应的操作。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-02 17:19:00" itemprop="dateCreated datePublished" datetime="2020-08-02T17:19:00+08:00">2020-08-02</time>
                </span>
                <span id="/9529.html" class="post-meta-item leancloud_visitors" data-flag-title="Airtest 和 Poco 的 API 总结" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>16k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>14 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9525.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9525.html" class="post-title-link" itemprop="url">Kubernetes 如何限制特定 namespace 的权限</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>最近遇到一个问题，那就是需要给别人共享一下 Kubernetes 的某个资源的使用和访问权限，这个仅仅存在于某个 namespace 下，但是我又不能把管理员权限全都给它，我想只给他授予这一个 Namespace 下的权限，那应该怎么办呢？ 比如我这边是需要只想授予 postgresql 这个 Namespace 的权限，这里我就需要利用到 Kubernetes 里面的 RBAC 机制来实现了，下面记录了我的操作流程。</p>
                  <h2 id="创建-Namespace"><a href="#创建-Namespace" class="headerlink" title="创建 Namespace"></a>创建 Namespace</h2>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">kubectl create <span class="keyword">namespace</span> <span class="symbol">postgresql</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先没有 Namespace 的话需要创建一个 Namespace，这里我创建的是 postgresql，大家可以自行修改。</p>
                  <h2 id="创建-ServiceAccount"><a href="#创建-ServiceAccount" class="headerlink" title="创建 ServiceAccount"></a>创建 ServiceAccount</h2>
                  <p>接下来需要创建一个 ServiceAccount，yaml 如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">apiVersion:</span> v1</span><br><span class="line"><span class="symbol">kind:</span> ServiceAccount</span><br><span class="line"><span class="symbol">metadata:</span></span><br><span class="line"><span class="symbol">  name:</span> postgresql</span><br><span class="line"><span class="symbol">  namespace:</span> postgresql</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="创建-Role"><a href="#创建-Role" class="headerlink" title="创建 Role"></a>创建 Role</h2>
                  <p>然后还要创建一个 Role，来控制相应的权限，yaml 如下：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">kind</span>: Role</span><br><span class="line"><span class="attribute">apiVersion</span>: rbac.authorization.k8s.io/v1beta1</span><br><span class="line"><span class="attribute">metadata</span>:</span><br><span class="line">  <span class="attribute">name</span>: postgresql</span><br><span class="line">  <span class="attribute">namespace</span>: postgresql</span><br><span class="line"><span class="attribute">rules</span>:</span><br><span class="line">  - <span class="attribute">apiGroups</span>: [<span class="string">""</span>, <span class="string">"extensions"</span>, <span class="string">"apps"</span>]</span><br><span class="line">    <span class="attribute">resources</span>: [<span class="string">"*"</span>]</span><br><span class="line">    <span class="attribute">verbs</span>: [<span class="string">"*"</span>]</span><br><span class="line">  - <span class="attribute">apiGroups</span>: [<span class="string">"batch"</span>]</span><br><span class="line">    <span class="attribute">resources</span>:</span><br><span class="line">      - jobs</span><br><span class="line">      - cronjobs</span><br><span class="line">    <span class="attribute">verbs</span>: [<span class="string">"*"</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里由于 Role 是 Namespace 级别的，所以只能在特定 Namespace 下生效，这里我要让授予本 Namespace 下的所有权限，这里 rules 就添加了所有的 API类型、资源类型和操作类型。</p>
                  <h2 id="创建-RoleBinding"><a href="#创建-RoleBinding" class="headerlink" title="创建 RoleBinding"></a>创建 RoleBinding</h2>
                  <p>最后需要将 Role 和 ServiceAccount 绑定起来，yaml 如下：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">kind</span>: RoleBinding</span><br><span class="line"><span class="attribute">apiVersion</span>: rbac.authorization.k8s.io/v1beta1</span><br><span class="line"><span class="attribute">metadata</span>:</span><br><span class="line">  <span class="attribute">name</span>: postgresql</span><br><span class="line">  <span class="attribute">namespace</span>: postgresql</span><br><span class="line"><span class="attribute">subjects</span>:</span><br><span class="line">  - <span class="attribute">kind</span>: ServiceAccount</span><br><span class="line">    <span class="attribute">name</span>: postgresql</span><br><span class="line">    <span class="attribute">namespace</span>: postgresql</span><br><span class="line"><span class="attribute">roleRef</span>:</span><br><span class="line">  <span class="attribute">apiGroup</span>: rbac.authorization.k8s.io</span><br><span class="line">  <span class="attribute">kind</span>: Role</span><br><span class="line">  <span class="attribute">name</span>: postgresql</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="创建-kubeconfig-文件"><a href="#创建-kubeconfig-文件" class="headerlink" title="创建 kubeconfig 文件"></a>创建 kubeconfig 文件</h2>
                  <p>现在我们执行上述 yaml 文件之后，ServiceAccount 其实就已经创建好了，它会对应一个 secret，我们来看下详情，执行：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">kubectl <span class="builtin-name">get</span> serviceaccount postgresql -n postgresql -o yaml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好，运行结果类似如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">apiVersion:</span> v1</span><br><span class="line"><span class="symbol">kind:</span> ServiceAccount</span><br><span class="line"><span class="symbol">metadata:</span></span><br><span class="line"><span class="symbol">  annotations:</span></span><br><span class="line">    kubectl.kubernetes.io/last-applied-configuration: |</span><br><span class="line">      &#123;<span class="string">"apiVersion"</span>:<span class="string">"v1"</span>,<span class="string">"kind"</span>:<span class="string">"ServiceAccount"</span>,<span class="string">"metadata"</span>:&#123;<span class="string">"annotations"</span>:&#123;&#125;,<span class="string">"name"</span>:<span class="string">"postgresql"</span>,<span class="string">"namespace"</span>:<span class="string">"postgresql"</span>&#125;&#125;</span><br><span class="line"><span class="symbol">  creationTimestamp:</span> <span class="string">"2020-07-30T16:10:38Z"</span></span><br><span class="line"><span class="symbol">  name:</span> postgresql</span><br><span class="line"><span class="symbol">  namespace:</span> postgresql</span><br><span class="line"><span class="symbol">  resourceVersion:</span> <span class="string">"17800240"</span></span><br><span class="line"><span class="symbol">  selfLink:</span> <span class="meta-keyword">/api/</span>v1<span class="meta-keyword">/namespaces/</span>postgresql<span class="meta-keyword">/serviceaccounts/</span>postgresql</span><br><span class="line"><span class="symbol">  uid:</span> <span class="number">6327</span>db1f<span class="number">-6</span>a93<span class="number">-4</span>f1e-b988<span class="number">-31842989</span>bbbc</span><br><span class="line"><span class="symbol">secrets:</span></span><br><span class="line">- name: postgresql-token-v26k7</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里它实际关联了一个 secret，叫做 postgresql-token-v26k7，这里面就隐藏了 ServiceAccount 的 token 和证书。 好，那么我们就可以利用这个 secret 来制作 kubeconfig 文件了，命令如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">server</span>=https://your-server:443</span><br><span class="line"><span class="attribute">name</span>=postgresql-token-v26k7</span><br><span class="line"><span class="attribute">namespace</span>=postgresql</span><br><span class="line"></span><br><span class="line"><span class="attribute">ca</span>=$(kubectl <span class="builtin-name">get</span> secret/<span class="variable">$name</span> -n <span class="variable">$namespace</span> -o <span class="attribute">jsonpath</span>=<span class="string">'&#123;.data.ca.crt&#125;'</span>)</span><br><span class="line"><span class="attribute">token</span>=$(kubectl <span class="builtin-name">get</span> secret/<span class="variable">$name</span> -n <span class="variable">$namespace</span> -o <span class="attribute">jsonpath</span>=<span class="string">'&#123;.data.token&#125;'</span> | base64 --decode)</span><br><span class="line"></span><br><span class="line">echo <span class="string">"apiVersion: v1</span></span><br><span class="line"><span class="string">kind: Config</span></span><br><span class="line"><span class="string">clusters:</span></span><br><span class="line"><span class="string">- name: test</span></span><br><span class="line"><span class="string">  cluster:</span></span><br><span class="line"><span class="string">    certificate-authority-data: <span class="variable">$&#123;ca&#125;</span></span></span><br><span class="line"><span class="string">    server: <span class="variable">$&#123;server&#125;</span></span></span><br><span class="line"><span class="string">contexts:</span></span><br><span class="line"><span class="string">- name: test</span></span><br><span class="line"><span class="string">  context:</span></span><br><span class="line"><span class="string">    cluster: test</span></span><br><span class="line"><span class="string">    user: postgresql</span></span><br><span class="line"><span class="string">current-context: test</span></span><br><span class="line"><span class="string">users:</span></span><br><span class="line"><span class="string">- name: postgresql</span></span><br><span class="line"><span class="string">  user:</span></span><br><span class="line"><span class="string">    token: <span class="variable">$&#123;token&#125;</span></span></span><br><span class="line"><span class="string">"</span> &gt; postgresql.kubeconfig</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们需要指定三个变量：</p>
                  <ul>
                    <li>server：就是 Kubernetes Server API 的地址</li>
                    <li>name：就是 ServiceAccount 对应的 secret</li>
                    <li>namespace：就是当前操作的 Namespace</li>
                  </ul>
                  <p>运行之后就会生成一个 portgresql.kubeconfig 文件。</p>
                  <h2 id="使用"><a href="#使用" class="headerlink" title="使用"></a>使用</h2>
                  <p>那么怎么使用呢？很简单，设置下环境变量切换下就好了。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="builtin-name">export</span> <span class="attribute">KUBECONFIG</span>=postgresql.kubeconfig</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们就将 KUBECONFIG 设置了下，这样再执行 kubectl 就会读取到当前的 kubeconfig 文件，就会生效了。 这时候我们来测试下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">kubectl <span class="builtin-name">get</span> nodes</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="builtin-name">Error</span> <span class="keyword">from</span><span class="built_in"> server </span>(Forbidden): nodes is forbidden:<span class="built_in"> User </span><span class="string">"system:serviceaccount:postgresql:postgresql"</span> cannot list<span class="built_in"> resource </span><span class="string">"nodes"</span> <span class="keyword">in</span> API<span class="built_in"> group </span><span class="string">""</span> at the cluster scope</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到这里就提示没有列出节点的权限了。 然后我们操作下 postgresql 下的权限试试：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">kubectl</span> <span class="meta">get</span> <span class="keyword">svc </span>-n postgresql</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">NAME                  TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)    AGE</span><br><span class="line">postgresql            ClusterIP   <span class="number">10.0</span><span class="number">.193</span><span class="number">.137</span>   &lt;none&gt;        <span class="number">5432</span>/TCP   <span class="number">9</span>d</span><br><span class="line">postgresql-headless   ClusterIP   None           &lt;none&gt;        <span class="number">5432</span>/TCP   <span class="number">9</span>d</span><br><span class="line">postgresql-metrics    ClusterIP   <span class="number">10.0</span><span class="number">.60</span><span class="number">.88</span>     &lt;none&gt;        <span class="number">9187</span>/TCP   <span class="number">9</span>d</span><br><span class="line">postgresql-read       ClusterIP   <span class="number">10.0</span><span class="number">.236</span><span class="number">.184</span>   &lt;none&gt;        <span class="number">5432</span>/TCP   <span class="number">9</span>d</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里就可以看到对 postgresql 这个命名空间的操作就成功了。 到此为止我们就成功实现了特定 Namespace 的限制，大功告成。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-02 17:16:52" itemprop="dateCreated datePublished" datetime="2020-08-02T17:16:52+08:00">2020-08-02</time>
                </span>
                <span id="/9525.html" class="post-meta-item leancloud_visitors" data-flag-title="Kubernetes 如何限制特定 namespace 的权限" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9522.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9522.html" class="post-title-link" itemprop="url">发布一个爬虫练习平台</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>之前也写过不少关于爬虫的博客了，比如我拿一个案例来写了一篇博客，当时写的时候好好的，结果过了一段时间这个页面改版了，甚至直接下线了，那这篇案例就废掉了。 另外如果拿别人的站或者 App 来做案例的话，比较容易触犯到对方的利益，风险比较高，比如把某个站的 JavaScript 逆向方案公布出来，比如把某个 App 的逆向方案公布出来。如果此时此刻没有对方联系你的话，一个很大原因可能是规模太小了别人没注意到，但不代表以后不会。我还是选择爱护自己的羽毛，关于逆向实际网站和 App 的案例我都不会发的。在这种情况下比较理想的方案是自建案例，只用这个案例讲明白对应的知识点就可以了。 所以，为此，这段时间我也在写一些爬虫相关的案例，比如：</p>
                  <ul>
                    <li>无反爬的服务端渲染网站</li>
                    <li>带有参数加密的 SPA 网站</li>
                    <li>各类形式验证码网站</li>
                    <li>反 WebDriver 网站</li>
                    <li>字体反爬等网站</li>
                    <li>模拟登录网站</li>
                    <li>App 案例，如代理检测，SSL Pining 等</li>
                  </ul>
                  <p>今天发布一下。</p>
                  <h2 id="案例列表"><a href="#案例列表" class="headerlink" title="案例列表"></a>案例列表</h2>
                  <p>本案例平台自爬数据、自建页面、自接反爬，案例稳定后永不过期，适合教学与练习。</p>
                  <h3 id="SSR-网站"><a href="#SSR-网站" class="headerlink" title="SSR 网站"></a>SSR 网站</h3>
                  <ul>
                    <li><a href="https://ssr1.scrape.center/" target="_blank" rel="noopener">ssr1</a>：猫眼电影数据网站，数据通过服务端渲染，适合基本爬虫练习。</li>
                    <li><a href="https://ssr2.scrape.center/" target="_blank" rel="noopener">ssr2</a>：HTTPS 无效证书网站，适合做跳过验证 HTTPS 案例。</li>
                    <li><a href="https://ssr3.scrape.center/" target="_blank" rel="noopener">ssr3</a>：HTTP Basic Authentication 网站，适合做 HTTP 认证案例，用户名密码均为 admin。</li>
                    <li><a href="https://ssr4.scrape.center/" target="_blank" rel="noopener">ssr4</a>：每个响应增加了 5 秒延迟，适合测试慢速网站爬取或做爬取速度测试，减少本身网速干扰。</li>
                  </ul>
                  <h3 id="SPA-网站"><a href="#SPA-网站" class="headerlink" title="SPA 网站"></a>SPA 网站</h3>
                  <ul>
                    <li><a href="https://spa1.scrape.center/" target="_blank" rel="noopener">spa1</a>：猫眼电影数据网站，数据通过 Ajax 加载，页面动态渲染，适合 Ajax 分析和动态页面渲染爬取。</li>
                    <li><a href="https://spa2.scrape.center/" target="_blank" rel="noopener">spa2</a>：猫眼电影数据网站，数据通过 Ajax 加载，数据接口参数加密且有时间限制，适合动态页面渲染爬取或 JavaScript 逆向分析。</li>
                    <li><a href="https://spa3.scrape.center/" target="_blank" rel="noopener">spa3</a>：猫眼电影数据网站，数据通过 Ajax 加载，无页码翻页，适合 Ajax 分析和动态页面渲染抓取。</li>
                    <li><a href="https://spa4.scrape.center/" target="_blank" rel="noopener">spa4</a>：新闻网站索引，数据通过 Ajax 加载，无页码翻页，适合 Ajax 分析和动态页面渲染抓取以及智能页面提取分析。</li>
                    <li><a href="https://spa5.scrape.center/" target="_blank" rel="noopener">spa5</a>：豆瓣图书网站，数据通过 Ajax 加载，有翻页，无反爬，适合大批量动态页面渲染抓取。</li>
                    <li><a href="https://spa6.scrape.center/" target="_blank" rel="noopener">spa6</a>：电影数据网站，数据通过 Ajax 加载，数据接口参数加密且有时间限制，源码经过混淆，适合 JavaScript 逆向分析。</li>
                  </ul>
                  <h3 id="验证码网站"><a href="#验证码网站" class="headerlink" title="验证码网站"></a>验证码网站</h3>
                  <ul>
                    <li><a href="https://captcha1.scrape.center/" target="_blank" rel="noopener">captcha1</a>：对接滑动拼图验证码，适合滑动拼图验证码分析处理。</li>
                    <li><a href="https://captcha2.scrape.center/" target="_blank" rel="noopener">captcha2</a>：对接图标点选验证码，适合图标点选验证码分析处理。</li>
                    <li><a href="https://captcha3.scrape.center/" target="_blank" rel="noopener">captcha3</a>：对接图文点选验证码，适合图文点选验证码分析处理。</li>
                    <li><a href="https://captcha4.scrape.center/" target="_blank" rel="noopener">captcha4</a>：对接语序分析验证码，适合语序分析验证码分析处理。</li>
                    <li><a href="https://captcha5.scrape.center/" target="_blank" rel="noopener">captcha5</a>：对接空间推理验证码，适合空间推理验证码分析处理。</li>
                    <li><a href="https://captcha6.scrape.center/" target="_blank" rel="noopener">captcha6</a>：对接九宫格识图验证码，适合九宫格识图验证码分析处理。</li>
                  </ul>
                  <h3 id="模拟登录网站"><a href="#模拟登录网站" class="headerlink" title="模拟登录网站"></a>模拟登录网站</h3>
                  <ul>
                    <li><a href="https://login1.scrape.center/" target="_blank" rel="noopener">login1</a>：登录时用户名和密码经过加密处理，适合 JavaScript 逆向分析。</li>
                    <li><a href="https://login2.scrape.center/" target="_blank" rel="noopener">login2</a>：对接 Session + Cookies 模拟登录，适合用作 Session + Cookies 模拟登录练习。</li>
                    <li><a href="https://login3.scrape.center/" target="_blank" rel="noopener">login3</a>：对接 JWT 模拟登录方式，适合用作 JWT 模拟登录练习。</li>
                  </ul>
                  <h3 id="反爬型网站"><a href="#反爬型网站" class="headerlink" title="反爬型网站"></a>反爬型网站</h3>
                  <ul>
                    <li><a href="https://antispider1.scrape.center/" target="_blank" rel="noopener">antispider1</a>：WebDriver 反爬网站，检测到 WebDriver 就不显示页面。</li>
                    <li><a href="https://antispider2.scrape.center/" target="_blank" rel="noopener">antispider2</a>：对接 User-Agent 反爬，检测到常见爬虫 User-Agent 就会拒绝响应，适合用作 User-Agent 反爬练习。</li>
                    <li><a href="https://antispider3.scrape.center/" target="_blank" rel="noopener">antispider3</a>：对接文字偏移反爬，所见顺序并不一定和源码顺序一致，适合用作文字偏移反爬练习。</li>
                    <li><a href="https://antispider4.scrape.center/" target="_blank" rel="noopener">antispider4</a>：对接字体文件反爬，显示的内容并不在 HTML 内，而是隐藏在字体文件，设置了文字映射表，适合用作字体反爬练习。</li>
                    <li><a href="https://antispider5.scrape.center/" target="_blank" rel="noopener">antispider5</a>：限制 IP 访问频率为最多 1 秒一个，如果过多则会封禁 IP。</li>
                  </ul>
                  <h3 id="App"><a href="#App" class="headerlink" title="App"></a>App</h3>
                  <ul>
                    <li><a href="https://app1.scrape.center/" target="_blank" rel="noopener">app1</a>：最基本的 App 案例，数据通过接口加载，无反爬，无任何加密参数，适合做抓包分析和请求模拟。</li>
                    <li><a href="https://app2.scrape.center/" target="_blank" rel="noopener">app2</a>：设置了接口请求不走系统代理，因此无法直接抓包，适合做抓包特殊处理。</li>
                    <li><a href="https://app3.scrape.center/" target="_blank" rel="noopener">app3</a>：对系统代理进行了检测，如果设置了代理则无法正常请求数据，适合做抓包特殊处理。</li>
                    <li><a href="https://app4.scrape.center/" target="_blank" rel="noopener">app4</a>：设置了 SSL Pining，如果设置了非法证书则无法正常请求数据，适合做反 SSL Pining 处理。</li>
                    <li><a href="https://app5.scrape.center/" target="_blank" rel="noopener">app5</a>：接口增加了加密参数，适合做抓包实时处理或可视化爬取或逆向分析。</li>
                    <li><a href="https://app6.scrape.center/" target="_blank" rel="noopener">app6</a>：接口增加了加密参数，同时对源码进行了混淆，适合做抓包实时处理或可视化爬取或逆向分析。</li>
                    <li><a href="https://app7.scrape.center/" target="_blank" rel="noopener">app7</a>：接口增加了加密参数，同时对安装包进行了加固处理，适合做抓包实时处理或可视化爬取或逆向分析。</li>
                  </ul>
                  <p>暂且是这么多，后续还会继续增加，大家可以试着爬爬看。</p>
                  <h2 id="汇总链接"><a href="#汇总链接" class="headerlink" title="汇总链接"></a>汇总链接</h2>
                  <p>为了方便，我专门申请了一个域名，scrape.center，意思名为「爬取中心」，似乎听起来意义上还说的过去？ 案例平台首页 URL：<a href="https://scrape.center" target="_blank" rel="noopener">https://scrape.center</a>，截图如下： <img src="https://qiniu.cuiqingcai.com/frxt1.png" alt="案例首页"> 大家可以点击任意一个网站来爬取练习。</p>
                  <h2 id="案例预览"><a href="#案例预览" class="headerlink" title="案例预览"></a>案例预览</h2>
                  <p>下面是一些部分案例的截图： <img src="https://qiniu.cuiqingcai.com/vpo8g.png" alt="SSR1案例截图"> <img src="https://qiniu.cuiqingcai.com/gpaen.png" alt="SPA4案例截图"> <img src="https://qiniu.cuiqingcai.com/p1tyv.png" alt="SPA5案例截图"> <img src="https://qiniu.cuiqingcai.com/om4z6.png" alt="SPA6案例截图"> 上面是一些案例的效果，基本上是使用 Django + Vue.js 开发的，主题使用了红色调，整个案例平台风格比较统一。另外还有一些 App 也是类似的风格，大家可以自行下载体验试试。 当然这里面最主要的还是案例的功能，比如各种加密、反爬、检测等等。</p>
                  <h2 id="源代码"><a href="#源代码" class="headerlink" title="源代码"></a>源代码</h2>
                  <p>有朋友可能会问这个案例平台的源代码在哪里。 这里解释一下，由于这个案例平台以后会用于案例的讲解，并且可能会出现在课程、书本中，所以为了避免盗版和抄袭的问题，这里我选择了闭源，也算是对自己的知识成果的另一种保护吧。 不过这并不意味着爬取代码是闭源的，这块还是会开源出来的。</p>
                  <h2 id="怎么爬"><a href="#怎么爬" class="headerlink" title="怎么爬"></a>怎么爬</h2>
                  <p>还有朋友会问，这一个个网站这么多类型和反爬，到底怎么爬呢？ 其实现在针对这个练习平台一些案例讲解我已经做好了，课程也基本 OK 了，如果感兴趣大家可以点击看看。 <a href="https://t.lagou.com/cRC3RGRjSu706" target="_blank" rel="noopener">https://t.lagou.com/cRC3RGRjSu706</a> 谢谢大家。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-08-02 17:10:16" itemprop="dateCreated datePublished" datetime="2020-08-02T17:10:16+08:00">2020-08-02</time>
                </span>
                <span id="/9522.html" class="post-meta-item leancloud_visitors" data-flag-title="发布一个爬虫练习平台" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9436.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9436.html" class="post-title-link" itemprop="url">浅谈NoSQL-Redis持久化存储，二者有何优缺点。又该如何选择？</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="Persistenced："><a href="#Persistenced：" class="headerlink" title="Persistenced："></a>Persistenced：</h1>
                  <h1 id="RDB（Redis-Database）："><a href="#RDB（Redis-Database）：" class="headerlink" title="RDB（Redis Database）："></a>RDB（Redis Database）：</h1>
                  <h3 id="简介："><a href="#简介：" class="headerlink" title="简介："></a>简介：</h3>
                  <p>将时间段间隔内的内存数据以快照的形式写入磁盘，它恢复时是将快照文件直接读到内存里（snapshot）</p>
                  <h3 id="原理："><a href="#原理：" class="headerlink" title="原理："></a>原理：</h3>
                  <p>Redis 会单独创建（fork）一个子进程来进行持久化，会先将数据写入到一个临时文件中，待持久化过程都结束了，在用这个临时文件替换上次持久化号的文件。<strong>主进程是不进行任何的IO操作，确保了极高的性能</strong>。如果需要进行大规模的数据恢复，且对于数据恢复的完整性不是非常敏感，那RDB方式要比AOF方式更加的高效。RDB的缺点是最后一次持久化后的数据可能会<strong>丢失</strong></p>
                  <h3 id="Fork："><a href="#Fork：" class="headerlink" title="Fork："></a>Fork：</h3>
                  <blockquote>
                    <p>作用：创建一个与当前进程一样的进程。新进程的所有数据（变量、环境变量、程序计数器等）都与原进程一致。但为一个全新的进程，并作为原进程的子进程</p>
                  </blockquote>
                  <p>RDB保存的是dump.rdb文件</p>
                  <h3 id="配置位置："><a href="#配置位置：" class="headerlink" title="配置位置："></a>配置位置：</h3>
                  <p>在redis.conf 中的snapshoting</p>
                  <h3 id="快照："><a href="#快照：" class="headerlink" title="快照："></a>快照：</h3>
                  <blockquote>
                    <p>save，save只管保存，其他不管全部阻塞 bgsave：redis会在后台异步进行快照操作 快照同时还可以响应客户端请求，可以通过lastsave命令获取最后一次成功执行快照的时间 执行flush命令也会产生dump.rdb文件，但里面是空的，无意义</p>
                  </blockquote>
                  <h3 id="如何恢复："><a href="#如何恢复：" class="headerlink" title="如何恢复："></a>如何恢复：</h3>
                  <blockquote>
                    <p>将备份文件（dump.rdb）移动到redis安装目录，并启动服务即可 在连接完成之后的终端 使用 <strong>config get dir</strong> 获取目录 异常恢复: redis-check-rdb —fix {}</p>
                  </blockquote>
                  <p>优势：</p>
                  <blockquote>
                    <p>适合大规模的数据备份 对数据完整性和一致性要求不高</p>
                  </blockquote>
                  <p>劣势：</p>
                  <blockquote>
                    <p>在一定时间间隔内做一次备份，所以如果redis意外down了就会丢失最后一次快照后的所有更改 Fork的时候，内存中的数据被克隆了一份，内存等将会2倍膨胀性，需考虑！！！</p>
                  </blockquote>
                  <h3 id="如何停止："><a href="#如何停止：" class="headerlink" title="如何停止："></a>如何停止：</h3>
                  <p>​ 动态所有停止rdb保存规则方法：redis-cli config set save “” </p>
                  <h1 id="AOF：（Append-only-File）："><a href="#AOF：（Append-only-File）：" class="headerlink" title="AOF：（Append only File）："></a>AOF：（Append only File）：</h1>
                  <h2 id="导入："><a href="#导入：" class="headerlink" title="导入："></a>导入：</h2>
                  <blockquote>
                    <p>为什么还会出现AOF？（新技术的出现必定弥补老技术的不足,新技术一定会借鉴老技术,是老技术的子类） 如果一个系统里面同时存在RDB是冲突呢还是协作? 为什么AOF会在RDB之后产生 AOF它会有什么优缺点?</p>
                  </blockquote>
                  <p>原理:<strong>以日志的形式来记录每个写操作</strong>,将Redis执行过的所有写<strong>指令记录下来</strong>(读操作不记录),只需追加文件但不可改写文件,Redis启动之初会读取该文件重新构建数据, 换言之,redis重启就根据日志文件的内容将写指令从前到后执行一次以完成数据恢复工作 配置位置: redis.conf中的APPEND ONLY MODE 配置说明: Appendfsunc:</p>
                  <blockquote>
                    <p>always: 同步持久化,每次发生更改立即记录到磁盘,性能差但数据完整性比较好 everysec:出厂默认推荐,异步操作,每秒记录,如果一秒内宕机,有数据丢失 No</p>
                  </blockquote>
                  <p>No-appendfsy-on-rewrite:重写时是否可以运用Appendfsync,默认no即可,保障数据安全 Auto-aof-rewrite-min-size :设定重写基准值 Auto-aof-rewrite-percentage : 设定重写基准值</p>
                  <h3 id="探讨dump-rdb-与aof的二者是否能共存及选择顺序"><a href="#探讨dump-rdb-与aof的二者是否能共存及选择顺序" class="headerlink" title="探讨dump.rdb,与aof的二者是否能共存及选择顺序"></a>探讨dump.rdb,与aof的二者是否能共存及选择顺序</h3>
                  <blockquote>
                    <p>当aof损坏时,rdb完全,二者可以和平共存 二者先选择aof</p>
                  </blockquote>
                  <h3 id="AOF启动／修复／恢复"><a href="#AOF启动／修复／恢复" class="headerlink" title="AOF启动／修复／恢复:"></a>AOF启动／修复／恢复:</h3>
                  <h4 id="正常恢复"><a href="#正常恢复" class="headerlink" title="正常恢复:"></a>正常恢复:</h4>
                  <p>启动:将redis.conf中APPEND ONLY MODE 下appendonly no改为yes 将有数据的aof文件复制一份保存到对于目录(config get dir) 恢复: 重启redis然后重新加载即可</p>
                  <h4 id="异常恢复"><a href="#异常恢复" class="headerlink" title="异常恢复:"></a>异常恢复:</h4>
                  <p>备份被写坏的AOF文件, 修复: Redistribution-check-aof —fix 进行修复 重启redis,重新加载即可</p>
                  <h3 id="Rewrite-重写"><a href="#Rewrite-重写" class="headerlink" title="Rewrite:重写"></a>Rewrite:重写</h3>
                  <blockquote>
                    <p>AOF采用文件追加的方式，文件会越来越大为避免出现此种情况，新增了重写机制， 当AOF文件的大小超过所设定的阈值时，Ｒｅｄｉｓ会启动AOF文件的内容压缩，只保留可以恢复数据的最小指令集，可以使用命令ｂｇｒｅｗｒｉｔｅａｏｆ</p>
                  </blockquote>
                  <h4 id="重新原理："><a href="#重新原理：" class="headerlink" title="重新原理："></a>重新原理：</h4>
                  <blockquote>
                    <p>AOF文件持续增长而过大时,会fork出一条新进程来讲文件重写(也是先写零食文件最后在rename),遍历新进程的内存数据,每条记录有一条的set语句.重写aof文件的操作,并没有读取旧的aof文件,而是将整个内存中的数据库内容用命令的方式重写了一个新的aof文件.和快照有点类似</p>
                  </blockquote>
                  <p>触发机制: REDIS会记录上次重写时的AOF大小,默认配置时当AOF文件大小是上次rewrite后大小的一倍且文件大于64M时触发 优势:</p>
                  <blockquote>
                    <p>每秒同步:appendfsync always 同步持久化 每次发送数据变更会被立即记录到磁盘,性能较差完整性比较友好 每修改同步: appendfsync everysec 异步操作,每秒记录,如果一秒内宕机,有数据丢失 不同步:appendfsync no 从不同步</p>
                  </blockquote>
                  <p>劣势: 相同数据集的数据而言aof文件要远大于rdb文件,恢复速度慢于rdb AOF运行效率要慢于rdb,每秒同步策略效率较好,不同步效率和rdb相同 </p>
                  <h2 id="Which-one-："><a href="#Which-one-：" class="headerlink" title="Which one?："></a>Which one?：</h2>
                  <p>RDB持久化方式:能够在指定的时间间隔内对数据进行快照存储 AOF持久化方式:记录每次对服务器写的操作,当服务器重启的时候会重新执行这些命令来恢复原始的数据,AOF命令以Redis协议追加保存每次写的操作到文件末尾,Redis还能对AOF文件进行后台重新,使得AOF文件的体积不至于过大 只做缓存:</p>
                  <blockquote>
                    <p>如果希望数据在服务器运行的时候存在,也可以不使用热河持久化方式.</p>
                  </blockquote>
                  <p>同时开启两种持久化方式:</p>
                  <blockquote>
                    <p>当redis重启的时候会优先载入AOF文件来恢复原始的数据,因为在通常情况下AOF文件保存的数据集要比rdb文件保存的数据集更完整 RDB的数据不实时,同时使用两者时服务器重启也指挥找AOF文件 那干脆直接使用AOF？不建议 因为RDB更适合用于数据库（AOF在不断变化不好备份），快速重启，而且不担保有AOF可能潜在的BUG，留着作为一个万一的手段</p>
                  </blockquote>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-23 11:06:07" itemprop="dateCreated datePublished" datetime="2020-07-23T11:06:07+08:00">2020-07-23</time>
                </span>
                <span id="/9436.html" class="post-meta-item leancloud_visitors" data-flag-title="浅谈NoSQL-Redis持久化存储，二者有何优缺点。又该如何选择？" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9511.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/9511.html" class="post-title-link" itemprop="url">浅谈排序算法与优化（仅部分，Updating）</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="浅谈排序算法与优化（仅部分，Updating）"><a href="#浅谈排序算法与优化（仅部分，Updating）" class="headerlink" title="浅谈排序算法与优化（仅部分，Updating）"></a>浅谈排序算法与优化（仅部分，Updating）</h1>
                  <p><strong><a href="https://github.com/Payne-Wu/Data-structure-and-algorithm" target="_blank" rel="noopener">欢迎查阅与star的源码</a></strong> <strong><strong>写在最前面，此文章少了各排序算法的对比，但多了一份由浅入深的个人理解，以及代码、及算法的优化的思路</strong></strong> <strong><strong>阅读文章约 需 5min</strong></strong></p>
                  <h4 id="列表排序"><a href="#列表排序" class="headerlink" title="列表排序"></a>列表排序</h4>
                  <p><strong>排序</strong></p>
                  <blockquote>
                    <p>将一组“无序”的记录序列调整为“有序”的记录序列</p>
                  </blockquote>
                  <p><strong>列表排序</strong></p>
                  <blockquote>
                    <p>将无序的列表变为有序列表 输入：列表； 输出：有序列表</p>
                  </blockquote>
                  <p>升序与降序 内置排序函数：sort（），基于timsort排序算法</p>
                  <blockquote>
                    <p>Timsort是一种混合稳定排序算法，源自归并排序（merge sort）和插入排序（insertion sort）</p>
                  </blockquote>
                  <p>有兴趣的伙计可以看看这两篇文章 <a href="https://blog.csdn.net/yangzhongblog/article/details/8184707" target="_blank" rel="noopener">sort算法运用原理1</a> <a href="https://www.cnblogs.com/brucecloud/p/6085703.html" target="_blank" rel="noopener">sort算法运用原理2</a> <img src="https://img-blog.csdnimg.cn/20200720020754601.gif#pic_center" alt="TimSort"></p>
                  <h4 id="常见排序算法"><a href="#常见排序算法" class="headerlink" title="常见排序算法"></a>常见排序算法</h4>
                  <h5 id="LOW："><a href="#LOW：" class="headerlink" title="LOW："></a>LOW：</h5>
                  <p><strong>冒泡：</strong></p>
                  <blockquote>
                    <p>列表每两个相邻的数，如果前面比后面大，则交换这两个数 一趟排序完成后，则无序区减少一个数，有序区增加一个数</p>
                    <figure class="highlight angelscript">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">@cal_time</span><br><span class="line">def bubble_sort(li: List[List]) -&gt; List:</span><br><span class="line">print(<span class="string">'初始列表'</span>, li)</span><br><span class="line">print(<span class="string">'初始列表长度：'</span>, len(li))</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> range(len(li) - <span class="number">1</span>):  # 第i躺,总次数</span><br><span class="line">i += <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> j <span class="keyword">in</span> range(len(li) - i - <span class="number">1</span>):  # 指针</span><br><span class="line">   <span class="keyword">if</span> li[j] &gt; li[j + <span class="number">1</span>]:  # 如果指针所对应的值大于对比的值，则二者交换位置</span><br><span class="line">       li[j], li[j + <span class="number">1</span>] = li[j + <span class="number">1</span>], li[j]</span><br><span class="line">   <span class="keyword">else</span>:</span><br><span class="line">       <span class="keyword">break</span></span><br><span class="line">print(f<span class="string">'冒泡排序第&#123;i&#125;次'</span>, li)</span><br><span class="line"></span><br><span class="line">li = [np.random.randint(<span class="number">0</span>, <span class="number">1000</span>) <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">5</span>)]</span><br><span class="line">bubble_sort(li)</span><br><span class="line"></span><br><span class="line">数据来源于numpy.randmo.randint()随机取数，运行流程：</span><br><span class="line">冒泡排序第<span class="number">1</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">2</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">3</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">4</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">5</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">6</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">7</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">8</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line">冒泡排序第<span class="number">9</span>次 [<span class="number">257</span>, <span class="number">620</span>, <span class="number">136</span>, <span class="number">379</span>, <span class="number">392</span>, <span class="number">118</span>, <span class="number">312</span>, <span class="number">892</span>, <span class="number">647</span>, <span class="number">655</span>]</span><br><span class="line"></span><br><span class="line"># 排序次数：列表长度<span class="number">-1</span>,</span><br><span class="line"># 缘由将无序区转化为有序区：(再看一遍以下这段代码， 如果li[j]&gt;li[j+<span class="number">1</span>],则二者交换位置）</span><br><span class="line"><span class="keyword">if</span> li[j] &gt; li[j + <span class="number">1</span>]: </span><br><span class="line">    li[j], li[j + <span class="number">1</span>] = li[j + <span class="number">1</span>], li[j]</span><br><span class="line"></span><br><span class="line"># 当一维数组长度为<span class="number">10000</span>的时候所需时间为：</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p><img src="https://img-blog.csdnimg.cn/20200720020710125.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d6cDcwODE=,size_16,color_FFFFFF,t_70#pic_center" alt="Runtime"> <strong>优化：</strong> 在一次回头看看冒泡排序具体实现思路，将<strong>列表每两个相邻的数，如果前面比后面大，则交换这两个数</strong>,<strong>一趟排序完成后，则无序区减少一个数，有序区增加一个数</strong>。 如果无序区已经是有序的呢？按照代码执行流程可知， 如果前者比后者大，那么则两个交换位置。（假设为降序，前面为无序区，后面为有序区） 否则不执行任何交换操作，但<strong>会执行便利</strong>（也可以理解为此运算”不执行任何交换操作，无实际意义”） 优化目标：(将“不执行任何交换的操作去掉”)，咱们在回头看看，具体实际的运算是在第二层for循环里面的，也就说所谓的“无用功”也是在这里产生的 无用功体现为：无论是否进行了位置交换，都会在往有序区在遍历检查一遍 思路如下： 主要优化的地方在跳出循环，在它不交换位置的时候，直接跳出此次的循环 假设它全部都是有序的，并设个标记True， 当如果循环内发生了位置交换，则改变标记为False。 流程控制，if。当if True的时则会执行if内部代码（设立return，或者break）主要跳出循环。避免对有序区进行有一次的排序操作 具体代码实现如下：</p>
                    <figure class="highlight stata">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">def bubble_sort(<span class="keyword">li</span>: <span class="keyword">List</span>):</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">"The len of List:"</span>, len(<span class="keyword">li</span>))  # 输入数组长度</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="keyword">range</span>(len(<span class="keyword">li</span>) - 1):  # 保证输入的数组在计算范围类，此处的“-1”是因为索引=长度-1</span><br><span class="line">        flag = True</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="keyword">range</span>(len(<span class="keyword">li</span>) - i - 1):</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">li</span>[j] &gt; <span class="keyword">li</span>[j + 1]:</span><br><span class="line">                <span class="keyword">li</span>[j + 1], <span class="keyword">li</span>[j] = <span class="keyword">li</span>[j], <span class="keyword">li</span>[j + 1]</span><br><span class="line">                flag = False</span><br><span class="line">            <span class="keyword">print</span>(<span class="string">"Sorting times:%d， Status：%s"</span> % (i + 1, <span class="keyword">li</span>))</span><br><span class="line">        <span class="keyword">if</span> flag:</span><br><span class="line">            <span class="keyword">break</span></span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                  </blockquote>
                  <p><strong>选择:</strong> <strong>插入:</strong></p>
                  <h5 id="Stronger"><a href="#Stronger" class="headerlink" title="Stronger"></a>Stronger</h5>
                  <blockquote>
                    <p>快速排序： 堆排序 归并排序</p>
                  </blockquote>
                  <h5 id="其他排序："><a href="#其他排序：" class="headerlink" title="其他排序："></a>其他排序：</h5>
                  <blockquote>
                    <p>希尔排序 计数排序 基数排序</p>
                  </blockquote>
                  <figure class="highlight kotlin">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">break</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><strong>选择:</strong> <strong>插入:</strong></p>
                  <h5 id="Stronger-1"><a href="#Stronger-1" class="headerlink" title="Stronger"></a>Stronger</h5>
                  <blockquote>
                    <p>快速排序： 堆排序 归并排序</p>
                  </blockquote>
                  <h5 id="其他排序：-1"><a href="#其他排序：-1" class="headerlink" title="其他排序："></a>其他排序：</h5>
                  <blockquote>
                    <p>希尔排序 计数排序 基数排序</p>
                  </blockquote>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-20 02:19:51" itemprop="dateCreated datePublished" datetime="2020-07-20T02:19:51+08:00">2020-07-20</time>
                </span>
                <span id="/9511.html" class="post-meta-item leancloud_visitors" data-flag-title="浅谈排序算法与优化（仅部分，Updating）" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9501.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9501.html" class="post-title-link" itemprop="url">听说B站弹幕鬼畜多？我不信</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p><a href="https://github.com/Mr2753/PythonScrape/tree/master/Scrape_BiliBili/BiliBli" target="_blank" rel="noopener">欢迎Star的源码</a></p>
                  <blockquote>
                    <p>新入手，小白的我，在我眼里Request爬虫永远只有四大步，不服来辩？</p>
                    <ol>
                      <li>确定URL，构造请求头</li>
                      <li>发送请求，获取响应</li>
                      <li>解析响应，获取数据</li>
                      <li>保存数据</li>
                    </ol>
                  </blockquote>
                  <h4 id="目标：根据视频BV，获取B站视频弹幕"><a href="#目标：根据视频BV，获取B站视频弹幕" class="headerlink" title="目标：根据视频BV，获取B站视频弹幕"></a>目标：根据视频BV，获取B站视频弹幕</h4>
                  <p>代码地址如下： 抓包确定URL：</p>
                  <blockquote>
                    <h5 id="导入："><a href="#导入：" class="headerlink" title="导入："></a>导入：</h5>
                    <p>视频都有一个唯一区分视频：BV号 那么视频的URL规则为：’<a href="https://wwww.bilibili.com/video/BV{BVID" target="_blank" rel="noopener">https://wwww.bilibili.com/video/BV{BVID</a>}’ 找一下弹幕的地址,直接search，即可！如下 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/07/list.so_.png" alt=""></p>
                    <p>由以上抓包可知，弹幕的URL：’<a href="https://api.bilibili.com/x/v1/dm/list.so?oid=oid" target="_blank" rel="noopener">https://api.bilibili.com/x/v1/dm/list.so?oid=oid</a>‘, 我们获取到oid那么这一步就完成了 来，回头去找一下oid从何而来呢？ 据老夫多年经验指引，他一定在视频URL里面。（其实当时也找了挺久的，甚至逆向那一手，断点调试、调用堆栈等等什么都用出来了。最终还是功夫不负有心人，找到了） 其实回头看，oid是等于video_URL页面里面的cid参数的（验证了Payne式猜想）。过程是难受的 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/07/Find_cid.png" alt=""></p>
                  </blockquote>
                  <p>URL，其参数规则也找到了，那么还不就随我为所欲为了。只要拿到视频地址，那不就可以直接拿到弹幕了么。of course！</p>
                  <h3 id="此处省略3万字（请求，解析，网络原理。。。）"><a href="#此处省略3万字（请求，解析，网络原理。。。）" class="headerlink" title="此处省略3万字（请求，解析，网络原理。。。）"></a>此处省略3万字（请求，解析，网络原理。。。）</h3>
                  <p>其实当时知道两个方法都去试了，JS那个就不说了，有兴趣的盆友，可以去搞一下 说说这个提取cid参数吧，我用的是正则，这种情况最好是用正则，不过也看个人喜好吧。 可以回头看第二张图，初一乍看我好像不会，啊哈哈~ </p>
                  <blockquote>
                    <p>经过优化后（主要是看了其他视频的那啥之后）：写出这个神奇的正则</p>
                    <figure class="highlight ini">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="attr">cid</span> = re.findall(<span class="string">'.*?cid":+(d&#123;9&#125;)+'</span>, text)[<span class="number">0</span>] </span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                  </blockquote>
                  <p> 谢谢你认真看到了末尾，那我也写点私活吧。欢迎查阅源码与star</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">其实关于很多网站（普通）的参数就算是有<span class="keyword">JS加密，以及混淆。并不代表就一定需要去解密，去逆向。有时候真的只需要serch一下惊喜多多，望诸君切记、切记。</span></span><br><span class="line"><span class="keyword">咱们解密，逆向，有时候完全即可做一个动态维持即可。例如我源码中的scrape </span>拉勾</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-15 22:48:07" itemprop="dateCreated datePublished" datetime="2020-07-15T22:48:07+08:00">2020-07-15</time>
                </span>
                <span id="/9501.html" class="post-meta-item leancloud_visitors" data-flag-title="听说B站弹幕鬼畜多？我不信" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>850</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9496.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/9496.html" class="post-title-link" itemprop="url">三探迭代、递归、循环</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h4 id="导入："><a href="#导入：" class="headerlink" title="导入："></a>导入：</h4>
                  <blockquote>
                    <h6 id="补充知识："><a href="#补充知识：" class="headerlink" title="补充知识："></a>补充知识：</h6>
                    <p>迭代、递归与循环：迭代与递归都是循环的子集，一个是取值推算，一个是不断的调用自己。 相同点：迭代、递归、循环都是“重复” 相似点：调用逻辑相似 不同点：我简单理解为迭代是根据自身的上一个值推算下一个值，而递归则是由上一个值与“己身”直接运算。循环是自身与外界计算 堆栈关系调用不同 当然也不能说谁好谁坏，只能说三者主针对不同</p>
                    <h5 id="来，来，来，翠花上栗子："><a href="#来，来，来，翠花上栗子：" class="headerlink" title="来，来，来，翠花上栗子："></a>来，来，来，翠花上栗子：</h5>
                    <p><strong>递归，归去来兮 ：</strong></p>
                    <figure class="highlight angelscript">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">def func(i):</span><br><span class="line">    i -= <span class="number">1</span></span><br><span class="line">    <span class="keyword">if</span> i == <span class="number">1</span>:</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        print(i)</span><br><span class="line">        <span class="keyword">return</span> func(i)</span><br><span class="line"></span><br><span class="line">print(func(<span class="number">8</span>))</span><br><span class="line"># 输出结果：<span class="number">7</span>、<span class="number">6</span>、<span class="number">5</span>、<span class="number">4</span>、<span class="number">3</span>、<span class="number">2</span>、<span class="number">1</span></span><br><span class="line">print(func(<span class="number">4</span>))</span><br><span class="line"># 输出结果：<span class="number">3</span>、<span class="number">2</span>、<span class="number">1</span></span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p><strong>迭代，更新换代</strong>（单单以数值方面考虑，凸显堆栈方面就。。。）</p>
                    <figure class="highlight php">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="comment"># 迭代</span></span><br><span class="line"><span class="keyword">List</span> = [<span class="string">'1'</span>,<span class="string">'2'</span>,<span class="string">'3'</span>]</span><br><span class="line">i = <span class="number">0</span></span><br><span class="line"><span class="keyword">while</span> <span class="keyword">True</span>:</span><br><span class="line">    <span class="keyword">print</span>（<span class="keyword">List</span>[i]）</span><br><span class="line">    i += <span class="number">1</span></span><br><span class="line"><span class="comment"># 循环、也有点递归的意思。小问题，小问题，这里探究迭代与循环</span></span><br><span class="line">count = <span class="number">0</span></span><br><span class="line"><span class="keyword">while</span> <span class="keyword">True</span>：</span><br><span class="line">  count = count + <span class="number">1</span></span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>个人思考及感悟：如何判断生成器与迭代器？：</p>
                    <figure class="highlight vim">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">区分循环、迭代</span><br><span class="line">循环：与外界计算，重复亦或者</span><br><span class="line"></span><br><span class="line">区分迭代器与生成器</span><br><span class="line">迭代器：</span><br><span class="line">可以被<span class="keyword">next</span>()函数调用并不断返回下一个值的对象称为迭代器</span><br><span class="line">生成器：</span><br><span class="line">不但可以作用于<span class="keyword">for</span>循环，还可以被<span class="keyword">next</span>()函数不断调用并返回下一个值，直到出现</span><br><span class="line">StopIteration</span><br><span class="line">可使用isinstance()方法判断是否是Iterator对象</span><br><span class="line"></span><br><span class="line">g = (i ** i <span class="keyword">for</span> i in <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">10</span>))</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">next</span>(g))</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">next</span>(g))</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">next</span>(g))</span><br><span class="line">。。。 。。。</span><br><span class="line"><span class="keyword">for</span> <span class="keyword">j</span> in <span class="variable">g:</span></span><br><span class="line">    <span class="keyword">print</span>(<span class="keyword">j</span>)</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                  </blockquote>
                  <h4 id="迭代器是什么？"><a href="#迭代器是什么？" class="headerlink" title="迭代器是什么？"></a>迭代器是什么？</h4>
                  <blockquote>
                    <p>迭代取值的工具，迭代是个循环，但不是重复的过程。每次的值都基于上次的值而来。迭代是特殊的重复</p>
                  </blockquote>
                  <h4 id="迭代器能做什么？"><a href="#迭代器能做什么？" class="headerlink" title="迭代器能做什么？"></a>迭代器能做什么？</h4>
                  <blockquote>
                    <p>迭代取值；程序中用的比较多就是先存后取 用时在释放。释放next()函数，以及遍历</p>
                  </blockquote>
                  <h4 id="迭代器怎么使用？"><a href="#迭代器怎么使用？" class="headerlink" title="迭代器怎么使用？"></a>迭代器怎么使用？</h4>
                  <blockquote>
                    <p>​ next（可迭代对象） 或者 可迭代对象. next () 以及在函数中yield关键字</p>
                    <figure class="highlight python">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">foo</span><span class="params">()</span>:</span></span><br><span class="line">    print(<span class="string">"starting..."</span>)</span><br><span class="line">    <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">        res = <span class="keyword">yield</span> <span class="number">4</span></span><br><span class="line">        print(<span class="string">"res:"</span>, res)</span><br><span class="line"></span><br><span class="line">g = foo()</span><br><span class="line">print(next(g))</span><br><span class="line">print(<span class="string">"*"</span> * <span class="number">20</span>)</span><br><span class="line">print(next(g))</span><br><span class="line">print(next(g))</span><br><span class="line">print(next(g))</span><br><span class="line"></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"><span class="string">starting...</span></span><br><span class="line"><span class="string">4</span></span><br><span class="line"><span class="string">********************</span></span><br><span class="line"><span class="string">res: None</span></span><br><span class="line"><span class="string">4</span></span><br><span class="line"><span class="string">res: None</span></span><br><span class="line"><span class="string">4</span></span><br><span class="line"><span class="string">res: None</span></span><br><span class="line"><span class="string">4</span></span><br><span class="line"><span class="string">'''</span></span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p><strong>带yield的函数是一个生成器，而不是一个函数了，这个生成器有一个函数就是next函数</strong></p>
                  </blockquote>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-15 04:53:07" itemprop="dateCreated datePublished" datetime="2020-07-15T04:53:07+08:00">2020-07-15</time>
                </span>
                <span id="/9496.html" class="post-meta-item leancloud_visitors" data-flag-title="三探迭代、递归、循环" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9489.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/9489.html" class="post-title-link" itemprop="url">重学算法与探究</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h4 id="写在最前面：本人也是个算法小白，如果有什么不对还望大佬指正，谢谢"><a href="#写在最前面：本人也是个算法小白，如果有什么不对还望大佬指正，谢谢" class="headerlink" title="写在最前面：本人也是个算法小白，如果有什么不对还望大佬指正，谢谢"></a>写在最前面：本人也是个算法小白，如果有什么不对还望大佬指正，谢谢</h4>
                  <h1 id="算法设计评价基本标准"><a href="#算法设计评价基本标准" class="headerlink" title="算法设计评价基本标准"></a>算法设计评价基本标准</h1>
                  <p>算法是对特定问题求解<strong>步骤</strong>的一种描述，如果将问题看作函数，那么算法是吧输入转化为输出</p>
                  <h4 id="算法"><a href="#算法" class="headerlink" title="算法:"></a>算法:</h4>
                  <p>是对特定问题求解步骤的一种描述，是为了解决一个或者一类问题给出的 一个确定的、有限长的操作序列。 算法的设计依赖于数据的存储结构，因此对确定的问题，应该需求子啊适宜的存储结构上设计出一种效率较高的算法</p>
                  <h4 id="算法的重要特性："><a href="#算法的重要特性：" class="headerlink" title="算法的重要特性："></a>算法的重要特性：</h4>
                  <h5 id="有穷性："><a href="#有穷性：" class="headerlink" title="有穷性："></a>有穷性：</h5>
                  <p>对于任何一组合法的输入值，在执行有穷步骤之后一定能结束，即算法中的操作步骤为有限个，并且每个步骤都能在有限的时间内完成</p>
                  <h5 id="确定性："><a href="#确定性：" class="headerlink" title="确定性："></a>确定性：</h5>
                  <p>对于每种情况下所应该执行的路径的操作，在算法中都有确切的规定，使算法的执行者或阅读者都能明确其含义及如何执行；并且在确切的条件下只有唯一一条执行流程路径</p>
                  <h5 id="可行性："><a href="#可行性：" class="headerlink" title="可行性："></a>可行性：</h5>
                  <p>算法中的所有操作都必须足够基本，都可以通过已经实现的基本运算执行有限次实现</p>
                  <h5 id="有输入"><a href="#有输入" class="headerlink" title="有输入:"></a>有输入:</h5>
                  <p>作为算法加工对象的量值，通常体现为算法中的一组变量。有些输入量需要在算法的执行过程中输入，而有些算法表面上没有输入，但实际上已被嵌入算法之中</p>
                  <h5 id="有输出："><a href="#有输出：" class="headerlink" title="有输出："></a>有输出：</h5>
                  <p>它是一组与“输入”有确定关系的量值，是算法进行信息加工够得到的结果。这种确定关系即为算法的功能</p>
                  <h4 id="算法设计目标："><a href="#算法设计目标：" class="headerlink" title="算法设计目标："></a>算法设计目标：</h4>
                  <h5 id="正确性："><a href="#正确性：" class="headerlink" title="正确性："></a>正确性：</h5>
                  <p><strong>算法应满足具体问题的需求，正确反映求解问题对输入、输出加工处理等方面的需求</strong></p>
                  <ol>
                    <li>程序中不含语法错误，计算的结果却不能满足规格说明要求。</li>
                    <li>程序对于特定的几组输入数据能够得出满足要求的结果，而对于其他的输入数据的不出正确的计算结果；</li>
                    <li>程序对于精心选择的、经典、苛刻且带有刁难性的几组输入数据能够得到满足需求的结果；（正确的算法）</li>
                    <li>程序对于一切合法的输入数据都能得出满足要求的结果</li>
                  </ol>
                  <h4 id="可读性："><a href="#可读性：" class="headerlink" title="可读性："></a>可读性：</h4>
                  <p><strong>算法处理用于编写程序子啊计算机上执行之外，另一个重要用处是阅读和交流。</strong> 算法中加入适当的注释，介绍算法的设计思路、各个模块的功能等一些必要性的说明文字来帮助读者理解算法。 <strong>要求:</strong> 算法中加入适当的注释，介绍算法的设计思路、各个模块的功能等一些必要性的说明文字来帮助读者理解算法 对算法中出现的各种自定义变量和类型能做到“见名知义”，即读者一看到某个变量（或类型名）就能知道其功能</p>
                  <h4 id="健壮性："><a href="#健壮性：" class="headerlink" title="健壮性："></a>健壮性：</h4>
                  <p>当输入数据非法时，算法能够适当地做出反应或进行处理，输出表示错误性质的信息并终止执行，而不会产生莫名其妙的输出结果。</p>
                  <h5 id="时间效率与存储占用量："><a href="#时间效率与存储占用量：" class="headerlink" title="时间效率与存储占用量："></a>时间效率与存储占用量：</h5>
                  <p>一般来说，求解同一个问题若有多种算法，则 执行时间短的算法效率高 占用存储空间少的算法较好 算法的执行时间开销和存储空间开销往往相互制约，对高时间效率和低存储占用的要求只能根据问题的性质折中处理</p>
                  <h1 id="算法复杂度"><a href="#算法复杂度" class="headerlink" title="算法复杂度"></a>算法复杂度</h1>
                  <h4 id="算法的时间复杂度："><a href="#算法的时间复杂度：" class="headerlink" title="算法的时间复杂度："></a>算法的时间复杂度：</h4>
                  <p>算法的效率指的是算法的执行时间随问题“规模”（通常用整型量n表示）的增长而增长的趋势 “规模”在此指的是输入量的数目，比如在排序问题中，问题的规模可以是被排序的元素数目 假如随着问题规模n的增长，算法执行时间的增长率和问题规模的增长率相同则可记为： T（n） = O（f(n)）</p>
                  <blockquote>
                    <p>f(n) 为问题规模n的某个函数； T（n）被称为算法的（渐进）时间复杂度（Time Complexity） O表示法不需要给出运行时间的精确值； 选择f（n），通常选择比较简单的函数形式，并忽略低次项和系数 常用的有O（1）、O（logn）、O（n）、O（nlogn）、O（n*n）等等 多项式时间复杂度的关系为： O(1) &lt; O(logn) &lt; O(n) &lt; O(N logn) &lt; O(n²) &lt; O(n³) 指数时间算法的关系为： O（2(n方)）&lt; O(n!) &lt;O(n(n方))</p>
                  </blockquote>
                  <p>由于估算算法时间复杂度关心的只是算法执行时间的增长率而不是绝对时间，因此可以忽略一些因素。 方法：从算法中选取一种对于所研究的问题来说是“<strong>基本操作”</strong>的原操作，以该“基本操作”在算法中重复执行的次数作为算法时间复杂度的依据。 EG：</p>
                  <blockquote>
                    <p>两个n x n 的矩阵相乘，求其时间复杂度</p>
                    <figure class="highlight ada">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="keyword">range</span>(<span class="number">10</span>):</span><br><span class="line">      <span class="keyword">for</span> j <span class="keyword">in</span> <span class="keyword">range</span>(<span class="number">10</span>): </span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>问题规模是矩阵的阶n,算法的控制结构式三重循环，基本操作是乘法操作 乘法执行次数为n³，则算法的时间复杂度为O(n³)</p>
                  </blockquote>
                  <h4 id="算法的空间复杂度："><a href="#算法的空间复杂度：" class="headerlink" title="算法的空间复杂度："></a>算法的空间复杂度：</h4>
                  <p>算法在执行期间所需要的存储量包括：</p>
                  <ol>
                    <li>程序代码所占用空间</li>
                    <li>输入数据所占用空间</li>
                    <li>辅助变量所占用的空间</li>
                  </ol>
                  <blockquote>
                    <p>为了降低复杂度，一个直观的思路是：梳理程序，看其流程中是否有无效的计算或者无效的存储。我们需要从时间复杂度和空间复杂度两个维度来考虑。 常用的降低时间复杂度的方法有递归、二分法、排序算法、动态规划等， 降低空间复杂度的方法，就要围绕数据结构做文章了。 降低空间复杂度的核心思路就是，能用低复杂度的数据结构能解决问题，就千万不要用高复杂度的数据结构。</p>
                  </blockquote>
                  <h4 id="如何评定一个程序算法的好坏？"><a href="#如何评定一个程序算法的好坏？" class="headerlink" title="如何评定一个程序算法的好坏？"></a>如何评定一个程序算法的好坏？</h4>
                  <p>简而言之：在符合算法设计标准的前提下，运行的更快、所用计算资源更少。即是更好的算法</p>
                  <blockquote>
                    <p>请注意这里的比较级别词<strong>更</strong>！！！，对于算法，个人认为就是获得同一结果的同时探究<strong>最优</strong>解决之道</p>
                  </blockquote>
                  <h5 id="探究算法优化（自我一点点小体悟-个人能力有限）"><a href="#探究算法优化（自我一点点小体悟-个人能力有限）" class="headerlink" title="探究算法优化（自我一点点小体悟-个人能力有限）"></a>探究算法优化（自我一点点小体悟-个人能力有限）</h5>
                  <blockquote>
                    <ol>
                      <li>
                        <p>抽象化：将<strong>不必要</strong>的计算过程去掉</p>
                        <blockquote>
                          <p>利用高斯算法解决累计求和问题</p>
                        </blockquote>
                      </li>
                      <li>
                        <p>慎选各<strong>数据结构</strong>，善用<strong>第三方</strong>算法</p>
                        <blockquote>
                          <p>去重： 善用迭代什么的等等 使用字典的特性-不可重复性 布隆过滤器去重（源于哈希算法）</p>
                        </blockquote>
                      </li>
                      <li>
                        <p>时空转换：将<strong>昂贵的</strong>时间转化为<strong>廉价的</strong>空间</p>
                        <blockquote>
                          <p>当‘优无可优时’,取贵舍廉 空间是可以用<strong>钱</strong>买的，加个什么<strong>内存</strong>啊，加个<strong>处理器</strong>等等什么的 时间是逝去就不在了</p>
                        </blockquote>
                        <p>以上仅仅个人的一点点小灵感，望大家不喜勿喷</p>
                      </li>
                    </ol>
                  </blockquote>
                  <p>说了这么多，来道简单的算法题目导入（同一计算机，i5, python） <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2020/07/电脑类型.png" alt=""> 需求如下：累计求和1-n的值(1. 为防止误差，验证10次; 2. 验证每次计算次数)</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># 在同一空间复杂下(也就是说没有迭代，探究暴力解法与高斯算法)</span></span><br><span class="line"><span class="comment"># n = 100000</span></span><br><span class="line"><span class="comment"># Method one 代码如下</span></span><br><span class="line">def func(n):</span><br><span class="line">    start = <span class="built_in">time</span>.<span class="built_in">time</span>()</span><br><span class="line">    <span class="built_in">count</span> = <span class="number">0</span></span><br><span class="line">    theSum = <span class="number">0</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> range(n + <span class="number">1</span>):</span><br><span class="line">        theSum = theSum + n</span><br><span class="line">        <span class="built_in">count</span> += <span class="number">1</span></span><br><span class="line">    <span class="keyword">end</span> = <span class="built_in">time</span>.<span class="built_in">time</span>()</span><br><span class="line"><span class="built_in">    return</span> theSum, <span class="keyword">end</span> - start, <span class="built_in">count</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">10</span>):</span><br><span class="line">    print(f<span class="string">"Sum is %d Required %10.10f seconds count = %d"</span> % func(<span class="number">10000</span>))</span><br><span class="line"><span class="comment"># %10.10f 表示取10位小数，运行结果如下</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <blockquote>
                    <p>Sum is 100010000 Required 0.0010061264 seconds count = 10001 Sum is 100010000 Required 0.0009891987 seconds count = 10001 Sum is 100010000 Required 0.0000000000 seconds count = 10001 Sum is 100010000 Required 0.0010235310 seconds count = 10001 Sum is 100010000 Required 0.0009710789 seconds count = 10001 Sum is 100010000 Required 0.0000000000 seconds count = 10001 Sum is 100010000 Required 0.0009973049 seconds count = 10001 Sum is 100010000 Required 0.0010013580 seconds count = 10001 Sum is 100010000 Required 0.0000000000 seconds count = 10001 Sum is 100010000 Required 0.0019786358 seconds count = 10001 探究可知：n扩大10倍，运算时间也会扩大10倍 高斯算法，从1累加至n,等于(首项+尾项)<em>项数/2 直接引用结论: 1+2+3+…+(n-1) +n={（1+n）+（2+（n-1））…}/2 = （n</em> （n + 1））/2</p>
                  </blockquote>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Gaussian算法，无迭代算法</span><br><span class="line">def fun1(n):</span><br><span class="line">    start = <span class="type">int</span>(<span class="type">time</span>.time())</span><br><span class="line">    theSum = (n * (n + <span class="number">1</span>)) / <span class="number">2</span></span><br><span class="line">    end = <span class="type">int</span>(<span class="type">time</span>.time())</span><br><span class="line">    <span class="keyword">return</span> theSum, <span class="keyword">end</span> - <span class="keyword">start</span>, n / <span class="number">2</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1000000</span>):</span><br><span class="line">    print(f"Sum is %d Required %10.100f seconds count = %d" % fun1(<span class="number">50</span>**<span class="number">100</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <blockquote>
                    <p><strong>计算结果如下：</strong> 循环计算10000次，出去I/O所需时间几乎可以不计。 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.0000000000000000000000000000000000000000000000000123123000000000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.000000000000000000000000000000000000000000000000012332100000000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.00000000000000000000000000000000000000000000000000000002310000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.000000000000000000000000000000000000000000000000000000010032000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.0000000000000000000000000000000000000000000000012231300000000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552 Sum is 49999999999999998486656110625518082973725163772751181324120875475173424217777037767098169202353125934013756207986941204091067867184139242319692520523619938935511795533394990905590906653083564427444224 Required 0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 seconds count = 5000000000000000079514455548799590234180404281972640694890663778873919386085190530406734992928407552</p>
                  </blockquote>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-14 02:51:59" itemprop="dateCreated datePublished" datetime="2020-07-14T02:51:59+08:00">2020-07-14</time>
                </span>
                <span id="/9489.html" class="post-meta-item leancloud_visitors" data-flag-title="重学算法与探究" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>7.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9484.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/9484.html" class="post-title-link" itemprop="url">网络爬虫工程师面试小笔记</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="网络爬虫工程师面试小笔记"><a href="#网络爬虫工程师面试小笔记" class="headerlink" title="网络爬虫工程师面试小笔记"></a>网络爬虫工程师面试小笔记</h1>
                  <p>————小企业，7K至10k版，面试总结。Payne</p>
                  <h4 id="面试题之一：Python单例模式"><a href="#面试题之一：Python单例模式" class="headerlink" title="面试题之一：Python单例模式"></a><strong>面试题之一：Python单例模式</strong></h4>
                  <ol>
                    <li>
                      <h5 id="什么是Python的单例模式？"><a href="#什么是Python的单例模式？" class="headerlink" title="什么是Python的单例模式？"></a>什么是Python的单例模式？</h5>
                    </li>
                  </ol>
                  <blockquote>
                    <p>单例模式（Singleton Pattern）是一种常用的软件设计模式，该模式主要目的是<strong>确保某一个类只有一个实例存在</strong>。当希望在整个系统中，某个类只能出现一个实例时，单例对象就派上用场了。 面向对象编程单例模式，保证了在程序运行中该类<strong>只实例化一次</strong>，并提供了一个全局访问点 Python的模块就是天然的单例模式 当模块在第一次导入时，就会生成 <strong>.pyc</strong> 文件 当第二次导入时就会<strong>直接先加载 .pyc</strong> 文件，而不会再次执行模块代码。 我们只需把相关函数和数据定义在一个模块中，就可以获得一个单例对象。</p>
                  </blockquote>
                  <ol>
                    <li>
                      <h5 id="如何实现单例模式？"><a href="#如何实现单例模式？" class="headerlink" title="如何实现单例模式？"></a>如何实现单例模式？</h5>
                    </li>
                  </ol>
                  <blockquote>
                    <h6 id="1-基于类："><a href="#1-基于类：" class="headerlink" title="1.基于类："></a><strong>1.基于类：</strong></h6>
                    <figure class="highlight python">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Singleton</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="comment"># def __new__(slef):类方法</span></span><br><span class="line">        <span class="comment"># pass </span></span><br><span class="line">        <span class="comment"># 当我们没写时默认调用object__new__方法</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 然后在执行类的实例化对象：__init__</span></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span> <span class="comment"># 实例方法</span></span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">instance</span><span class="params">(cls, *args,**kwargs)</span>:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> has attr(Singleton,<span class="string">"_instance"</span>):</span><br><span class="line">            Singleton._instance = Singleton(*args,**kwargs)</span><br><span class="line">        <span class="keyword">return</span> Singleton._intance</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>此时是以完成了一个简单的单例模式<strong>案例</strong>，But实际开发中随时凉凉 举例说明：</p>
                    <figure class="highlight reasonml">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="keyword">class</span> <span class="constructor">Singleton(<span class="params">object</span>)</span>:</span><br><span class="line"></span><br><span class="line">    def <span class="constructor">__init__(<span class="params">self</span>)</span>:</span><br><span class="line">        pass</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    def instance(cls, *args, **kwargs):</span><br><span class="line">        <span class="keyword">if</span> not hasattr(Singleton, <span class="string">"_instance"</span>):</span><br><span class="line">            <span class="module-access"><span class="module"><span class="identifier">Singleton</span>.</span><span class="module"><span class="identifier">_instance</span> </span></span>= <span class="constructor">Singleton(<span class="operator">*</span><span class="params">args</span>, <span class="operator">**</span><span class="params">kwargs</span>)</span></span><br><span class="line">        return <span class="module-access"><span class="module"><span class="identifier">Singleton</span>.</span><span class="module"><span class="identifier">_instance</span></span></span></span><br><span class="line"><span class="module-access"><span class="module"></span></span></span><br><span class="line"><span class="module-access"><span class="module">i</span></span>mport threading</span><br><span class="line"></span><br><span class="line">def task(arg):</span><br><span class="line">    obj = <span class="module-access"><span class="module"><span class="identifier">Singleton</span>.</span></span>instance<span class="literal">()</span></span><br><span class="line">    print(obj)</span><br><span class="line"></span><br><span class="line">for i <span class="keyword">in</span> range(<span class="number">10</span>):</span><br><span class="line">    t = threading.<span class="constructor">Thread(<span class="params">target</span>=<span class="params">task</span>,<span class="params">args</span>=[<span class="params">i</span>,])</span></span><br><span class="line">    t.start<span class="literal">()</span></span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>当然此时也并没有什么问题，BUT在’ init ‘方法中加入I/O（input/output）操作就凉凉了 问题出现了，按照以上方式创建的单例无法支持<strong>多线程</strong> 缘由：Python中实例化对象与初始化对象是分开执行的，又由于多线程之间是通信共享的，故出现线程安全问题。主要体现为，create一个之后kill一个，create一个又被kill一个。所以就。。。 解决思路一：相互独立，分而治之。加锁独立 也就是咱们所了解、知道的线程锁的概念，使得其无序变为相对有序。具体代码便不在此赘述 在看看思路一(相互独立，分而治之。加锁独立) 解决思路二：‘反’实例化，加锁保护独立，确保通用性 在Python3中，调用父类方法是为super()，那么是否可以增加判断： 当类属性不为空时，我们便不在实例化且返回一个<strong>已实例化</strong>的类属性。这样还是不太完美，带有局限性。进一步加锁保护优化以保障多线程情况下只有一个线程同时访问。这样就保障了单例的安全 <strong>基于 new 方法实现！！！</strong></p>
                    <figure class="highlight markdown">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">在回到基于类的第一个代码块，并详细查看其注释。</span><br><span class="line">实例化一个对象是先执行了类的<span class="strong">__new__</span>方法（若未写执行object.<span class="strong">__new__</span>），实例化对象；然后子啊执行类的<span class="strong">__init__</span>方法，对这个对象进行初始化。基于此实现单例。</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <ol>
                      <li>
                        <h6 id="基于装饰器"><a href="#基于装饰器" class="headerlink" title="基于装饰器"></a>基于装饰器</h6>
                      </li>
                    </ol>
                    <p>使用装饰器实现，实例如下：</p>
                    <figure class="highlight ruby">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">Singleton</span><span class="params">(cls)</span></span><span class="symbol">:</span></span><br><span class="line">    _instance = &#123;&#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_singleton</span><span class="params">(*args, **kargs)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> cls <span class="keyword">not</span> <span class="keyword">in</span> <span class="symbol">_instance:</span></span><br><span class="line">            _instance[cls] = cls(*args, **kargs)</span><br><span class="line">        <span class="keyword">return</span> _instance[cls]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> _singleton</span><br><span class="line"></span><br><span class="line">@Singleton</span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">A</span>(<span class="title">object</span>):</span></span><br><span class="line">    a = <span class="number">1</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, x=<span class="number">0</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.x = x</span><br><span class="line"></span><br><span class="line">a1 = A(<span class="number">2</span>)</span><br><span class="line">a2 = A(<span class="number">3</span>)</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <ol>
                      <li>
                        <h6 id="使用模块的方法"><a href="#使用模块的方法" class="headerlink" title="使用模块的方法"></a>使用模块的方法</h6>
                      </li>
                    </ol>
                    <p>书写代码(并保存在Singleton.py中)：</p>
                    <figure class="highlight angelscript">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="keyword">class</span> <span class="symbol">Singleton</span>(<span class="symbol">object</span>):</span><br><span class="line">  <span class="symbol">def</span> <span class="symbol">func</span>(<span class="symbol">self</span>):</span><br><span class="line">        <span class="symbol">pass</span></span><br><span class="line"><span class="symbol">singleton</span> = <span class="symbol">Sinleton</span>()</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>// from * import singleton 需使用时，直接在其他文件中导入此文件中的对象，那么这个对象即是单例模式对象 还有个基于元类的就没书写了具体请看：<a href="https://blog.csdn.net/weixin_44239343/article/details/89376796" target="_blank" rel="noopener">https://blog.csdn.net/weixin_44239343/article/details/89376796</a></p>
                  </blockquote>
                  <h4 id="面试题之二：Redis有几种数据类型？"><a href="#面试题之二：Redis有几种数据类型？" class="headerlink" title="面试题之二：Redis有几种数据类型？"></a>面试题之二：Redis有几种数据类型？</h4>
                  <blockquote>
                    <p>如果是单单是Redis那么<strong>常用数据类型为五种</strong> 他们分别是：String，List，hash，set，zset String：字符串，一个字符串Value最多可以是512M Hash：哈希，是一个String类型的field和Value的映射表 List：列表，时间是链表 Set：集合是一个String类型无序无重复集合，其通过Hash Table实现 Zset（sorted）：有序集合 那么应聘时，请注意这个小坑，你<strong>Python</strong>使用Redis又几种数据类型? 这个是基于语言来回答的，所用语言+Redis数据类型杂糅 Number,String，list，tuple（这个不确定），dict，aggregate 同时又涉语言所拥有的数据类型与redis，一样的就<strong>‘合二为一’</strong>嘛 以Python为例，稍后继续探究这（6+5）之间的杂糅，dict与aggregate其二者区别为主（其实我也不晓得更深的了）。以及1对1，1对多，多对1。数据结构搞起来，然后哼哼~。</p>
                  </blockquote>
                  <h4 id="面试题之三：Scrapy框架的运行流程及各模块的作用"><a href="#面试题之三：Scrapy框架的运行流程及各模块的作用" class="headerlink" title="面试题之三：Scrapy框架的运行流程及各模块的作用"></a>面试题之三：Scrapy框架的运行流程及各模块的作用</h4>
                  <blockquote>
                    <p>如果简历里面写了分布式会拓展scrapy-Redis架构以及其作用。 CAP理论，估计会扯到数据这块。拓展database什么特性啊，之类的。谈优化，谈数据结构。反正数据结构与算法这块，基于此，难于此，也凉于此</p>
                  </blockquote>
                  <h4 id="面试题之四：scrapy去重所用的几种机制"><a href="#面试题之四：scrapy去重所用的几种机制" class="headerlink" title="面试题之四：scrapy去重所用的几种机制"></a>面试题之四：scrapy去重所用的几种机制</h4>
                  <p>谨记：先从scrapy本身的去重原理及机制说起来，最基础，优缺点，去重原理等等。一步步来，一上来就BloonFilter，风险不小啊 对于此，自我总结如下：</p>
                  <blockquote>
                    <p>1、scrapy 基于内存</p>
                    <figure class="highlight routeros">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">scrapy源码中可以找到一个dupefilters.py去重器;</span><br><span class="line"></span><br><span class="line">需要将dont_filter设置为<span class="literal">False</span>开启去重，默认是<span class="literal">True</span>，没有开启去重；</span><br><span class="line"></span><br><span class="line">对于每一个url的请求，调度器都会根据请求得相关信息加密得到一个指纹信息，并且将指纹信息和<span class="builtin-name">set</span>()集合中的指纹信息进 行 比对，如果<span class="builtin-name">set</span>()集合中已经存在这个数据，就不在将这个Request放入队列中;</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>2、redis 基于内存 更加快捷、速度快、易于管理</p>
                    <figure class="highlight plain">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">不说了，前面是叩门砖，Redis就是决胜之地，没啥可讲的，会的基本都会，不会的我也不会，在深挖其原理数据结构，估计得喝上一点，也怕自己一不小心怕给扯飞了</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>3、布隆过滤器 大 可能存在拥有一定的错误率</p>
                    <figure class="highlight plain">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line">加分项，是满分还是SSS+。</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p><strong>对于此面试个人总结如下：</strong> 源于基础，死于基础（数据结构及类型， 以及算法） 知识点：点串线，线成面。 自己也还有很长一段路走，加油，加油~</p>
                  </blockquote>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Payne" class="author" itemprop="url" rel="index">Payne</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-07 03:57:28" itemprop="dateCreated datePublished" datetime="2020-07-07T03:57:28+08:00">2020-07-07</time>
                </span>
                <span id="/9484.html" class="post-meta-item leancloud_visitors" data-flag-title="网络爬虫工程师面试小笔记" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9479.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9479.html" class="post-title-link" itemprop="url">发布一个智能解析框架 Gerapy Auto Extractor</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>之前我写过几篇文章介绍过有关爬虫的智能解析算法，包括商业化应用 Diffbot、Readability、Newspaper 这些库，另外我有一位朋友之前还专门针对新闻正文的提取算法 GeneralNewsExtractor，这段时间我也参考和研究了一下这些库的算法，同时参考一些论文，也写了一个智能解析库，在这里就做一个非正式的介绍。</p>
                  <h2 id="引入"><a href="#引入" class="headerlink" title="引入"></a>引入</h2>
                  <p>那首先说说我想做的是什么。 比如这里有一个网站，网易新闻，<a href="https://news.163.com/rank/，这里有个新闻列表，预览图如下" target="_blank" rel="noopener">https://news.163.com/rank/，这里有个新闻列表，预览图如下</a>： <img src="https://qiniu.cuiqingcai.com/bt2xa.png" alt="image-20200705212920277"> 任意点开一篇新闻，看到的结果如下： <img src="https://qiniu.cuiqingcai.com/p0fyb.png" alt="image-20200705213058853"> 我现在需要做到的是在不编写任何 XPath、Selector 的情况下实现下面信息的提取： 对于列表页来说，我要提取新闻的所有标题列表和对应的链接，它们就是图中的红色区域： <img src="https://qiniu.cuiqingcai.com/feupo.png" alt="image-20200705213401471"> 这里红色区域分了多个区块，比如这里一共就是 40 个链接，我都需要提取出来，包括标题的名称，标题的 URL。 我们看到页面里面还有很多无用的链接，如上图绿色区域，包括分类、内部导航等，这些需要排除掉。 对于详情页，我主要关心的内容有标题、发布时间、正文内容，它们就是图中红色区域： <img src="https://qiniu.cuiqingcai.com/er8ay.png" alt="image-20200705213723198"> 其中这里也带有一些干扰项，比如绿色区域的侧边栏的内容，无用的分享链接等。 总之，我想实现某种算法，实现如上两大部分的智能化提取。</p>
                  <h2 id="框架"><a href="#框架" class="headerlink" title="框架"></a>框架</h2>
                  <p>之前我开发了一个叫做 Gerapy <a href="https://github.com/Gerapy/Gerapy" target="_blank" rel="noopener">https://github.com/Gerapy/Gerapy</a> 的框架，是一个基于 Scrapy、Scrapyd 的分布式爬虫管理框架，属 1.x 版本。现在正在开发 Gerapy 2.x 版本，其定位转向了 Scrapy 的可视化配置和调试、智能化解析方向，放弃支持 Scraypd，转而支持 Docker、Kubernetes 的部署和监控。 对于智能解析来说，就像刚才说的，我期望的就是上述的功能，在不编写任何 XPath 和 Selector 的情况下实现页面关键内容的提取。 框架现在发布了第一个初步版本，名称叫做 Gerapy Auto Extractor，名字 Gerapy 相关，也会作为 Gerapy 的其中一个模块。 GitHub 链接：<a href="https://github.com/Gerapy/GerapyAutoExtractor" target="_blank" rel="noopener">https://github.com/Gerapy/GerapyAutoExtractor</a> 现在已经发布了 PyPi，<a href="https://pypi.org/project/gerapy-auto-extractor/，可以使用" target="_blank" rel="noopener">https://pypi.org/project/gerapy-auto-extractor/，可以使用</a> pip3 来安装，安装方式如下：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> gerapy-<span class="keyword">auto</span>-extractor</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完了之后我们就可以导入使用了。</p>
                  <h2 id="功能"><a href="#功能" class="headerlink" title="功能"></a>功能</h2>
                  <p>下面简单介绍下它的功能，它能够做到列表页和详情页的解析。 列表页：</p>
                  <ul>
                    <li>标题内容</li>
                    <li>标题链接</li>
                  </ul>
                  <p>详情页：</p>
                  <ul>
                    <li>标题</li>
                    <li>正文</li>
                    <li>发布时间</li>
                  </ul>
                  <p>先暂时实现了如上内容的提取，其他字段的提取暂时还未实现。</p>
                  <h2 id="使用"><a href="#使用" class="headerlink" title="使用"></a>使用</h2>
                  <p>要使用 Gerapy Auto Extractor，前提我们必须要先获得 HTML 代码，注意这个 HTML 代码是我们在浏览器里面看到的内容，是整个页面渲染完成之后的代码。在某些情况下如果我们简单用「查看源代码」或 requests 请求获取到的源码并不是真正渲染完成后的 HTML 代码。 要获取完整 HTML 代码可以在浏览器开发者工具，打开 Elements 选项卡，然后复制你所看到的 HTML 内容即可。 先测试下列表页，比如我把 <a href="https://news.163.com/rank/" target="_blank" rel="noopener">https://news.163.com/rank/</a> 这个保存为 list.html， <img src="https://qiniu.cuiqingcai.com/rsxk6.png" alt="image-20200705220428754"> 然后编写提取代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import json</span><br><span class="line"><span class="keyword">from</span> gerapy_auto_extractor.extractors.list import extract_list</span><br><span class="line"></span><br><span class="line">html = open(<span class="string">'list.html'</span>, <span class="attribute">encoding</span>=<span class="string">'utf-8'</span>).read()</span><br><span class="line"><span class="builtin-name">print</span>(json.dumps(extract_list(html), <span class="attribute">indent</span>=2, <span class="attribute">ensure_ascii</span>=<span class="literal">False</span>, <span class="attribute">default</span>=str))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就是这么简单，核心代码就一行，就是调用了一个 extract_list 方法。 运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">"1家6口5年"</span>结离婚<span class="string">"10次:儿媳"</span>嫁<span class="string">"公公岳"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://news.163.com/20/0705/05/FGOFE1HJ0001875P.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">""</span>港独<span class="string">"议员泼水阻碍教科书议题林郑月娥深夜斥责"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://news.163.com/20/0705/02/FGO66FU90001899O.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">"感动中国致敬留德女学生:街头怒怼"</span>港独<span class="string">"有理有"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://news.163.com/20/0705/08/FGOPG3AM0001899O.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">"香港名医讽刺港警流血少过月经受访时辩称遭盗号"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://news.163.com/20/0705/01/FGO42EK90001875O.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">"李晨独居北京复式豪宅没想到肌肉男喜欢小花椅子"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://home.163.com/20/0705/07/FGOLER1200108GL2.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">"不战东京!林丹官宣退役正式结束20年职业生涯"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://sports.163.com/20/0704/12/FGML920300058782.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">"香港美女搬运工月薪1.6万每月花6千租5平出租"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://home.163.com/20/0705/07/FGOLEL1100108GL2.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  &#123;</span><br><span class="line">    <span class="attr">"title"</span>: <span class="string">"杭州第一大P2P"</span>凉了<span class="string">":近百亿未还!被警方立案"</span>,</span><br><span class="line">    <span class="attr">"url"</span>: <span class="string">"https://money.163.com/20/0705/07/FGON5T7B00259DLP.html"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  ...</span><br><span class="line">]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到想要的内容就提取出来了，结果是一个列表，包含标题内容和标题链接两个字段，由于内容过长，这里就省略了一部分。 接着我们再测试下正文的提取，随便打开一篇文章，比如 <a href="https://news.ifeng.com/c/7xrdz0kysde，保存下" target="_blank" rel="noopener">https://news.ifeng.com/c/7xrdz0kysde，保存下</a> HTML，命名为 detail.html。 <img src="https://qiniu.cuiqingcai.com/cjnaf.png" alt="image-20200705222433951"> 编写测试代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import json</span><br><span class="line"><span class="keyword">from</span> gerapy_auto_extractor.extractors import extract_detail</span><br><span class="line">html = open(<span class="string">'detail.html'</span>, <span class="attribute">encoding</span>=<span class="string">'utf-8'</span>).read()</span><br><span class="line"><span class="builtin-name">print</span>(json.dumps(extract_detail(html), <span class="attribute">indent</span>=2, <span class="attribute">ensure_ascii</span>=<span class="literal">False</span>, <span class="attribute">default</span>=str))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"title"</span>: <span class="string">"内蒙古巴彦淖尔发布鼠疫疫情Ⅲ级预警"</span>,</span><br><span class="line">  <span class="attr">"datetime"</span>: <span class="string">"2020-07-05 18:54:15"</span>,</span><br><span class="line">  <span class="attr">"content"</span>: <span class="string">"2020年7月4日，乌拉特中旗人民医院报告了1例疑似腺鼠疫病例，根据《内蒙古自治区鼠疫疫情预警实施方案》（内鼠防应急发﹝2020﹞7号）和《自治区鼠疫控制应急预案（2020年版）》（内政办发﹝2020﹞17号）的要求，经研究决定，于7月5日发布鼠疫防控Ⅲ级预警信息如下：n一、预警级别及起始时间n预警级别：Ⅲ级。n2020年7月5日起进入预警期，预警时间从本预警通告发布之日持续到2020年底。n二、注意事项n当前我市存在人间鼠疫疫情传播的风险，请广大公众严格按照鼠疫防控“三不三报”的要求，切实做好个人防护，提高自我防护意识和能力。不私自捕猎疫源动物、不剥食疫源动物、不私自携带疫源动物及其产品出疫区；发现病（死）旱獭及其他动物要报告、发现疑似鼠疫病人要报告、发现不明原因的高热病人和急死病人要报告。要谨慎进入鼠疫疫源地，如有鼠疫疫源地的旅居史，出现发热等不适症状时及时赴定点医院就诊。n按照国家、自治区鼠疫控制应急预案的要求，市卫生健康委将根据鼠疫疫情预警的分级，及时发布和调整预警信息。n巴彦淖尔市卫生健康委员会n2020年7月5日n来源：巴彦淖尔市卫生健康委员会"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>成功输出了标题、正文、发布时间等内容。 这里就演示了基本的列表页、详情页的提取操作。</p>
                  <h2 id="算法"><a href="#算法" class="headerlink" title="算法"></a>算法</h2>
                  <p>整个算法的实现比较杂，我看了几篇论文和几个项目的源码，然后经过一些修改实现的。 其中列表页解析的参考论文：</p>
                  <ul>
                    <li><a href="http://www.cnki.com.cn/Article/CJFDTotal-JSYJ201509023.htm" target="_blank" rel="noopener">面向不规则列表的网页数据抽取技术的研究</a></li>
                    <li><a href="https://kns.cnki.net/kcms/detail/detail.aspx?filename=PZKX201708010&amp;dbcode=CJFQ&amp;dbname=CJFD2017&amp;v=" target="_blank" rel="noopener">基于块密度加权标签路径特征的Web新闻在线抽取</a></li>
                  </ul>
                  <p>详情页解析的参考论文和项目：</p>
                  <ul>
                    <li><a href="https://kns.cnki.net/KCMS/detail/detail.aspx?dbcode=CJFQ&amp;dbname=CJFDLAST2019&amp;filename=GWDZ201908029&amp;v=MDY4MTRxVHJXTTFGckNVUkxPZmJ1Wm5GQ2poVXJyQklqclBkTEc0SDlqTXA0OUhiWVI4ZVgxTHV4WVM3RGgxVDM=" target="_blank" rel="noopener">基于文本及符号密度的网页正文提取方法</a></li>
                    <li><a href="https://github.com/kingname/GeneralNewsExtractor" target="_blank" rel="noopener">GeneralNewsExtractor</a></li>
                  </ul>
                  <p>这些都是不完全参考，然后加上自己的一些修改最终才形成了现在的结果。 算法在这里就几句话描述一下思路，暂时先不展开讲了。 列表页解析：</p>
                  <ul>
                    <li>找到具有公共父节点的连续相邻子节点，父节点作为候选节点。</li>
                    <li>根据节点特征进行聚类融合，将符合条件的父节点融合在一起。</li>
                    <li>根据节点的特征、文本密度、视觉信息（尚未实现）挑选最优父节点。</li>
                    <li>从最优父节点内根据标题特征提取标题。</li>
                  </ul>
                  <p>详情页解析：</p>
                  <ul>
                    <li>标题根据 meta、title、h 节点综合提取</li>
                    <li>时间根据 meta、正则信息综合提取</li>
                    <li>正文根据文本密度、符号密度、视觉信息（尚未实现）综合提取。</li>
                  </ul>
                  <p>后面等完善了之后再详细介绍算法的具体实现，现在如感兴趣可以去看源码。</p>
                  <h2 id="说明"><a href="#说明" class="headerlink" title="说明"></a>说明</h2>
                  <p>本框架仅仅发布了最初测试版本，测试覆盖度比较少，目前仅仅测试了有限的几个网站，尚未大规模测试和添加对比实验，因此准确率现在还没有标准的保证。 参考：关于详情页正文的提取我主要参考了 <a href="https://github.com/kingname/GeneralNewsExtractor" target="_blank" rel="noopener">GeneralNewsExtractor</a> 这个项目，原项目据测试可以达到 90% 以上的准确率。 列表页我测试了腾讯、网易、知乎等都是可以顺利提取的，如： <img src="https://qiniu.cuiqingcai.com/9cqi4.png" alt="19841593922229_.pic_hd"> <img src="https://qiniu.cuiqingcai.com/d0bsb.png" alt="image-20200705224404571"> <img src="https://qiniu.cuiqingcai.com/deq3p.png" alt="image-20200705224419759"> 后面会有大规模测试和修正。 项目初版，肯定存在很多不足，希望大家可以多发 Issue 和提 PR。 另外这里建立了一个 Gerapy 开发交流群，之前在 QQ 群的也欢迎加入，以后交流就在微信群了，大家在使用过程遇到关于 Gerapy、Gerapy Auto Extractor 的问题欢迎交流。 这里放一个临时二维码，后期可能会失效，失效后大家可以到公众号「进击的Coder」获取加群方式。 <img src="https://qiniu.cuiqingcai.com/fhh5v.png" alt="image-20200705225922008"></p>
                  <h2 id="待开发功能"><a href="#待开发功能" class="headerlink" title="待开发功能"></a>待开发功能</h2>
                  <ul>
                    <li>视觉信息的融合</li>
                    <li>文本相似度的融合</li>
                    <li>分类模型的融合</li>
                    <li>下一页翻页的信息提取</li>
                    <li>正文图片、视频的提取</li>
                    <li>对接 Gerapy</li>
                  </ul>
                  <p>最后感谢大家的支持！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-05 23:42:06" itemprop="dateCreated datePublished" datetime="2020-07-05T23:42:06+08:00">2020-07-05</time>
                </span>
                <span id="/9479.html" class="post-meta-item leancloud_visitors" data-flag-title="发布一个智能解析框架 Gerapy Auto Extractor" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9476.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/9476.html" class="post-title-link" itemprop="url">Mac 终端 Git Status 显示中文乱码</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>最近遇到 Mac 的 Git Status 显示中文乱码的问题，类似：</p>
                  <figure class="highlight taggerscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">"<span class="symbol">\3</span>43<span class="symbol">\2</span>00<span class="symbol">\2</span>20<span class="symbol">\3</span>45<span class="symbol">\2</span>67<span class="symbol">\2</span>45<span class="symbol">\3</span>44<span class="symbol">\2</span>75<span class="symbol">\2</span>34<span class="symbol">\3</span>43<span class="symbol">\2</span>00<span class="symbol">\2</span>21<span class="symbol">\3</span>45<span class="symbol">\2</span>17<span class="symbol">\2</span>21<span class="symbol">\3</span>51<span class="symbol">\2</span>02<span class="symbol">\2</span>56<span class="symbol">\3</span>44<span class="symbol">\2</span>73<span class="symbol">\2</span>66<span class="symbol">\3</span>46<span class="symbol">\2</span>34<span class="symbol">\2</span>15<span class="symbol">\3</span>45<span class="symbol">\2</span>12<span class="symbol">\2</span>41.md"</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>解决方案：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">git<span class="built_in"> config </span>--global core.quotepath <span class="literal">false</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>完了之后就显示正常了。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2020-07-04 02:28:11" itemprop="dateCreated datePublished" datetime="2020-07-04T02:28:11+08:00">2020-07-04</time>
                </span>
                <span id="/9476.html" class="post-meta-item leancloud_visitors" data-flag-title="Mac 终端 Git Status 显示中文乱码" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>197</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/6/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/6/">6</a><span class="page-number current">7</span><a class="page-number" href="/page/8/">8</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/8/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
